def commit_files(user, session):
    """ Moves compiled yang moudles to user's yang directory """

    directory = ServerSettings.session_path(session)
    if not os.path.exists(directory):
        logging.error('Session storage %s does not exist' % directory)
        return False, None

    yangdst = ServerSettings.yang_path(user)
    cxmldst = ServerSettings.cxml_path(user)
    count = 0

    if not os.path.exists(yangdst):
        logging.debug('Created ' + yangdst)
        os.makedirs(yangdst)

    if not os.path.exists(cxmldst):
        logging.debug('Created ' + cxmldst)
        os.makedirs(cxmldst)

    modules = ET.Element('modules')
    for cxmlpath in glob.glob(os.path.join(directory, '*.xml')):
        basename = os.path.basename(cxmlpath)
        if basename == 'dependencies.xml':
            continue
        base = os.path.splitext(basename)[0]
        yang_src_path = os.path.join(directory, base + '.yang')
        yang_dst_path = os.path.join(yangdst, base + '.yang')
        cxml_dst_path = os.path.join(cxmldst, base + '.xml')
        logging.debug('Committing ' + yang_src_path)
        if os.path.exists(yang_src_path):
            logging.debug('Commit ' + yang_dst_path)
            _clean_oldfiles(yangdst, base)
            _clean_oldfiles(cxmldst, base)
            os.rename(yang_src_path, yang_dst_path)
            os.rename(cxmlpath, cxml_dst_path)
            module = ET.Element('module')
            module.text = base + '.yang'
            modules.append(module)
            count += 1

    # There is a update in yang modules, delete existing dependency file
    # so that it will be recompiled next time
    if count > 0:
        session_d = os.path.join(directory, 'dependencies.xml')
        if os.path.exists(session_d):
            logging.debug('Moving dependency file ...')
            os.rename(session_d, os.path.join(yangdst, 'dependencies.xml'))
        else:
            logging.debug('Compiling user dependency ...')
            Compiler.compile_pyimport(user)

        # added module might affect existing module, recompile them
        _compile_dependecies(user, [m.text for m in modules], None)

    logging.debug('Committed ' + str(count) + ' file(s)')
    return True, modules

    def test_04_upload_handler(self):
        """ Verify upload_handler API in views.py """

        # 1. Login to django server
        response = self.chrome.post('/explorer/login/', {'username': 'demo', 'password': 'demo123', 'action': 'login'})
        self.assertTrue(response.status_code == 200)

        # 2. Intialize upload request
        response = self.chrome.get('/explorer/upload', {'mode' : 'init'})
        self.assertTrue(response.status_code == 200)
        self.assertTrue('ok' in response.content)

        # Get path to upload file
        curr = os.getcwd()
        pathlist = curr.split('/')
        while pathlist[-1] != 'yang-explorer': pathlist.pop()
        pathlist.append('default-models')
        root = '/'.join(pathlist)

        # 3. Upload file content
        filelist = ['ietf-inet-types.yang', 'ietf-yang-types.yang', '[email protected]']
        for _file in filelist:
            with open(os.path.join(root, _file), 'r') as fp:
                response = self.chrome.post('/explorer/upload-add', {'Filedata' : fp})
                self.assertTrue(response.status_code == 200)
                self.assertTrue('ok' in response.content)
                f = response.content.split('<module>')[1].split('</module>')[0]
                self.upload_files.append(f.strip())

        # 4. Compile file
        for _file in self.upload_files:
            response = self.chrome.get('/explorer/upload', {'mode':'sync', 'file': _file})
            self.assertTrue(response.status_code == 200)
            self.assertTrue('ok' in response.content)

        # 5. Commit file
        response = self.chrome.get('/explorer/upload', {'mode':'commit'})
        self.assertTrue(response.status_code == 200)
        self.assertTrue('ok' in response.content)

        # logout
        response = self.chrome.post('/explorer/login/', {'username': 'demo', 'password': 'demo123', 'action': 'logout'})
        self.assertTrue(response.status_code == 200)
        self.assertTrue('ok' in response.content)

        # Verify that files are actually uploaded
        yang_path = ServerSettings.yang_path('demo')
        cxml_path = ServerSettings.cxml_path('demo')
        for _file in self.upload_files:
            # check if yang file is uploaded
            self.assertTrue(os.path.exists(os.path.join(yang_path, _file)))

            # check if xml file is created
            xml_name = _file.split('.yang')[0] + '.xml'
            self.assertTrue(os.path.exists(os.path.join(cxml_path, xml_name)))

        print("Test: upload_handler PASSED")

def upload_handler(request):
    """ HTTP Request handler function to upload yang models """

    mode = request.GET.get('mode', '')
    logging.debug(request.method + ':Received upload request .. ' + mode)

    if not request.user.is_authenticated():
        logging.warning('User must be logged in !!')
        return HttpResponse(Response.error(mode, 'Unauthorized'))

    if not ServerSettings.user_aware():
        if not request.user.has_perm('explorer.delete_yangmodel') or \
                not request.user.has_perm('explorer.change_yangmodel'):
            logging.warning('Unauthorized upload request .. ')
            return HttpResponse(Response.error(mode, 'User does not have permission to upload !!'))

    if request.method == 'POST':
        # create a temporary storage for this session
        directory = ServerSettings.session_path(request.session.session_key)
        _file = Uploader.upload_file(request.FILES['Filedata'], directory)
        if _file is not None:
            module = ET.Element('module')
            module.text = _file
            rval = Response.success('upload', 'ok', xml=module)
            logging.debug(rval)
            return HttpResponse(rval)
        return HttpResponse(Response.error('upload', 'Failed to upload'))
    elif request.method == 'GET':
        if mode == 'sync':
            filename = request.GET.get('file', '')
            index = request.GET.get('index', '')
            logging.info('Received sync request for ' + filename + ', index ' + index)
            success, response = Uploader.sync_file(request.user.username,
                                                   request.session.session_key,
                                                   filename, index)
            if success:
                return HttpResponse(Response.success(mode, 'ok'))
            return HttpResponse(Response.error(mode, 'compilation failed', xml=response))

        elif mode == 'commit':
            success, modules = Uploader.commit_files(request.user.username, request.session.session_key)
            if success:
                return HttpResponse(Response.success('commit', 'ok', modules))

        elif mode == 'init':
            success, modules = Uploader.get_upload_files(request.user.username, request.session.session_key)
            if success:
                return HttpResponse(Response.success(mode, 'ok', modules))

        elif mode == 'clear':
            success, modules = Uploader.clear_upload_files(request.user.username, request.session.session_key)
            if success:
                return HttpResponse(Response.success(mode, 'ok', modules))
        return HttpResponse(Response.error(mode, 'failed'))

    return render_to_response('upload.html')

    def get_dependencies(username, modules, session):
        """
        return dependencies for given yang models
        """
        session_dir = ''
        logging.debug("get_dependencies: Target Modules " + str(modules))
        if session is not None:
            session_dir = ServerSettings.session_path(session)
            dfile = os.path.join(session_dir, 'dependencies.xml')
        else:
            dfile = os.path.join(ServerSettings.yang_path(username), 'dependencies.xml')

        if not os.path.exists(dfile):
            logging.error('get_dependencies: dependency file %s missing!!', dfile)
            return []

        if session_dir:
            session_files = [os.path.basename(_file) for _file in glob.glob(os.path.join(session_dir, '*.yang'))]

        yang_path = ServerSettings.yang_path(username)
        yang_files = [os.path.basename(_file) for _file in glob.glob(os.path.join(yang_path, '*.yang'))]

        dmodules = Set([])
        dgraph = DYGraph(dfile)
        for m in modules:
            module = dgraph.dependency_module(m)
            if module is None:
                continue
            for name in module.imports:
                dmodules.add(name)
            for name in module.includes:
                dmodules.add(name)
            for name in module.depends:
                dmodules.add(name)

        dmodules_list = list(dmodules)

        deplist = []
        for _file in dmodules_list:
            # prefer freshly uploaded files
            if session_dir:
                depfile = _find_matching(_file, session_dir, session_files)
            else:
                depfile = _find_matching(_file, yang_path, yang_files)

            if depfile is not None:
                deplist.append(depfile)
            else:
                logging.warning("get_dependencies: Dependency (%s) not satisfied, compilation will fail !!" %  _file)

        logging.debug("get_dependencies: Computed " + str(deplist))
        return deplist

def download_schema(request, req):
    '''
    This API download yang schema from device and bundle it
    '''
    logging.debug('Download Schemas')

    modules = download_yang(request, req)    

    session_dir = ServerSettings.schema_path(request.session.session_key)
    http_host = request.META['HTTP_HOST']
    current = str(datetime.now())
    current = current.replace(' ', '-')
    current = current[:current.rindex('.')]
    zfname = 'schema-' + current + '.zip'
    zfile = session_dir + '/' + zfname

    homedir = os.getcwd()
    os.chdir(session_dir)
    with ZipFile(zfile, "w") as lz:
        for f in glob.glob("*.yang"):
            lz.write(f)
            os.remove(f)
    if not lz.namelist():
        os.remove(zfile)
    os.chdir(homedir)

    url = '\nhttp://' + http_host + '/' + 'download/session/' + request.session.session_key + '/' + zfname
    return HttpResponse(Response.success('download', msg=url))

def download_schema(request, req):
    """
    This API download yang schema from device and bundle it
    """
    logger.debug("Download Schemas")

    modules = download_yang(request, req)

    session_dir = ServerSettings.schema_path(request.session.session_key)
    http_host = request.META["HTTP_HOST"]
    current = str(datetime.now())
    current = current.replace(" ", "-")
    current = current[: current.rindex(".")]
    zfname = "schema-" + current + ".zip"
    zfile = session_dir + "/" + zfname

    homedir = os.getcwd()
    os.chdir(session_dir)
    with ZipFile(zfile, "w") as lz:
        for f in glob.glob("*.yang"):
            lz.write(f)
            os.remove(f)
    if not lz.namelist():
        os.remove(zfile)
    os.chdir(homedir)

    url = "\nhttp://" + http_host + "/" + "download/session/"
    url += request.session.session_key + "/" + zfname
    return HttpResponse(Response.success("download", msg=url))

def login_handler(request):
    """ HTTP Request handler function for user login / logout requests """
    session = ET.Element('session')
    if request.POST:
        action = request.POST['action']
        if action == 'login':
            username = request.POST['username']
            password = request.POST['password']
            user = authenticate(username=username, password=password)
            if user is not None and user.is_active:
                # Correct password, and the user is marked "active"
                login(request, user)
                session.text = username
            else:
                return HttpResponse(Response.error('login', 'Authentication Failed'))
        else:
            try:
                if request.session.session_key is not None and request.session.session_key != '':
                    session_dir = ServerSettings.session_path(request.session.session_key)
                    if os.path.exists(session_dir):
                        logging.debug('Cleaning ' + session_dir)
                        shutil.rmtree(session_dir)
                logout(request)
            except Exception as ex:
                print(ex.__doc__)
                print(ex.message)
        logging.debug('Login success')
        return HttpResponse(Response.success(action, 'ok', session))
    return HttpResponse(Response.error(action, 'Invalid Request'))

def login_handler(request):
    """ HTTP Request handler function for user login / logout requests """
    if request.POST:
        action = request.POST['action']
        if action == 'login':
            username = request.POST['username']
            password = request.POST['password']
            user = authenticate(username=username, password=password)
            if user is not None and user.is_active:
                # Correct password, and the user is marked "active"
                login(request, user)
            else:
                return HttpResponse(Response.error('login', 'Authentication Failed'))
        else:
            username = ''
            try:
                if request.session.session_key is not None and request.session.session_key != '':
                    session_dir = ServerSettings.session_path(request.session.session_key)
                    if os.path.exists(session_dir):
                        logger.debug('Cleaning ' + session_dir)
                        shutil.rmtree(session_dir)
                logout(request)
            except:
                logger.exception("Failed")
            else:
                logger.debug('Logout success!!')
        session = get_session_config(username)
        return HttpResponse(Response.success(action, 'ok', session))
    return HttpResponse(Response.error('unknown', 'Invalid request!!'))

 def cxml_path(username, modulename):
     _dir = ServerSettings.cxml_path(username)
     modules = [os.path.basename(_file) for _file in glob.glob(os.path.join(_dir, '*.xml'))]
     for module in modules:
         if module == modulename + '.xml':
             return os.path.join(_dir, module)
         if module.startswith(modulename + '@'):
             return os.path.join(_dir, module)
     return None

def is_browsable(username, module):
    cxml_path = os.path.join(ServerSettings.cxml_path(username), module + '.xml')
    browsable = False
    if os.path.exists(cxml_path):
        try:
            root = ET.parse(cxml_path).getroot()
            if root.find('node'):
                browsable = True
        except:
            logger.error('is_browsable: Exception in parse -> ' + cxml_path)
    return browsable

def validate_schema(user, name, version):
    if version is None:
        fn = os.path.join(ServerSettings.yang_path(user), name + '.yang')
    else:
        fn = os.path.join(ServerSettings.yang_path(user), name+'@'+version+'.yang')
    if os.path.exists(fn):
        return None 

    dirpath = os.path.join(settings.BASE_DIR, ServerSettings.yang_path(user))
    sfile = os.path.basename(fn.split('@')[0])
    if not sfile.endswith('.yang'): 
        sfile = sfile + '.yang'

    for file in os.listdir(dirpath):
        yfile = os.path.basename(file.split('@')[0])
        if not yfile.endswith('.yang'):
            yfile = yfile + '.yang'
        if sfile == yfile: 
            return '[out-of-sync]'
    return '[not-exist]'

    def compile_pyimport(username, session=None):
        """
        Compile yang model and return tuple (boolean, list-of-errors)
        """
        plugins = os.path.join(settings.BASE_DIR, "explorer", "plugins")
        if not os.path.exists(plugins):
            logging.error("CXML Plugin directory is missing .. !!")
            return False, None

        if subprocess.call(["which", "pyang"]) != 0:
            logging.error("Could not find pyang compiler, please install pyang .. !!")
            return False, None

        logging.debug("Rebuilding dependencies for user %s" % username)

        # build include path
        includes = [ServerSettings.yang_path(username)]
        if session is not None:
            session_dir = ServerSettings.session_path(session)
            if not os.path.exists(session_dir):
                logging.error("compile_pyimport: Session directory %s not found !!", session_dir)
                return False, ["Session error !!"]
            includes.append(session_dir)
            depfile = os.path.join(session_dir, "dependencies.xml")
        else:
            depfile = os.path.join(includes[0], "dependencies.xml")

        target_yangs = []
        for yang_dir in includes:
            for _file in glob.glob(os.path.join(yang_dir, "*.yang")):
                target_yangs.append(_file)

        if not target_yangs:
            logging.debug("compile_pyimport: No yang file found !!")
            return True, ET.Element("messages")

        command = ["pyang", "-f", "pyimport", "--plugindir", "explorer/plugins", "-p"]
        command += [":".join(includes)]
        command += target_yangs

        return Compiler.invoke_compile(command, depfile)

    def compile_pyimport(username, session=None):
        '''
        Compile yang model and return tuple (boolean, list-of-errors)
        '''
        plugins = os.path.join(settings.BASE_DIR, 'explorer', 'plugins')
        if not os.path.exists(plugins):
            logging.error('CXML Plugin directory is missing .. !!')
            return (False, None)

        if subprocess.call(['which', 'pyang']) != 0:
            logging.error('Could not find pyang compiler, please install pyang .. !!')
            return (False, None)

        logging.debug('Rebuilding dependencies for user %s' % username)

        # build include path
        includes = [ServerSettings.yang_path(username)]
        if session is not None:
            session_dir = ServerSettings.session_path(session)
            if not os.path.exists(session_dir):
                logging.error('compile_pyimport: Session directory %s not found !!',  session_dir)
                return (False, ["Session error !!"])
            includes.append(session_dir)
            depfile = os.path.join(session_dir, 'dependencies.xml')
        else:
            depfile = os.path.join(includes[0], 'dependencies.xml')

        target_yangs = []
        for yang_dir in includes:
            for _file in glob.glob(os.path.join(yang_dir, '*.yang')):
                target_yangs.append(_file)

        if not target_yangs:
            logging.debug('compile_pyimport: No yang file found !!')
            return (True, ET.Element('messages'))

        command = ['pyang', '-f', 'pyimport', '--plugindir', 'explorer/plugins', '-p']
        command += [':'.join(includes)]
        command += target_yangs

        return Compiler.invoke_compile(command, depfile)

def validate_schema(user, name, version):
    if not version:
        fn = os.path.join(ServerSettings.yang_path(user), name + ".yang")
    else:
        fn = os.path.join(ServerSettings.yang_path(user), name + "@" + version + ".yang")

    if os.path.exists(fn):
        return None

    dirpath = os.path.join(settings.BASE_DIR, ServerSettings.yang_path(user))
    sfile = os.path.basename(fn.split("@")[0])
    if not sfile.endswith(".yang"):
        sfile = sfile + ".yang"

    for file in os.listdir(dirpath):
        yfile = os.path.basename(file.split("@")[0])
        if not yfile.endswith(".yang"):
            yfile = yfile + ".yang"
        if sfile == yfile:
            return "[out-of-sync]"
    return "[not-exist]"

def clear_upload_files(user, session):
    """ Delete uploaded yang files which are not committed """

    directory = ServerSettings.session_path(session)
    if not os.path.exists(directory):
        logging.debug('Session storage %s does not exist' % directory)
        return False, None

    modules = ET.Element('modules')
    for _file in glob.glob(os.path.join(directory, '*')):
        os.remove(_file)

    return True, modules

def sync_file(user, session, filename, index):
    """ Compile yang module """
    if index == '0':
        logging.debug('Compiling session dependency ...')
        (rc, msg) = Compiler.compile_pyimport(user, session)
        if not rc:
            return rc, msg

    _file = os.path.join(ServerSettings.session_path(session), filename)
    if os.path.exists(_file):
        (rc, msgs) = Compiler.compile_cxml(user, session, _file)
    else:
        logging.error('sync_file: File %s not found ' % filename)
        (rc, msgs) = (False, None)
    return rc, msgs

def get_upload_files(user, session):
    """ Get the list of uploaded yang files which are not committed """

    modules = ET.Element('modules')
    directory = ServerSettings.session_path(session)
    if not os.path.exists(directory):
        logging.error('Session storage %s does not exist' % directory)
        return True, modules

    for _file in glob.glob(os.path.join(directory, '*.yang')):
        module = ET.Element('module')
        module.text = os.path.basename(_file)
        modules.append(module)

    return True, modules

    def get_modulelist(username):
        """
        Return list of modules available to user
        """
        users = User.objects.filter(username=username)
        if not users:
            logger.warning("ModuleAdmin.admin_action: Invalid user " + username)
            return []

        modules = []
        files = glob.glob(os.path.join(ServerSettings.cxml_path(username), '*.xml'))
        for _file in files:
            module = os.path.basename(_file).split('.xml')[0]
            if UserProfile.objects.filter(user=users[0], module=module).exists():
                modules.append(module)
        return modules

def get_annotation_tree():
    ann_path = ServerSettings.annotation_path(None)
    filename = os.path.join(ann_path, 'covered.json')
    if not os.path.exists(filename):
        return None

    tree = cache.get('ui_ann', None)
    if tree is None:
        tree = XPathTree('/', None)
        with open(filename, 'r') as f:
            profile = json.load(f)
            for line in profile.get('data', []):
                tree.insert(line, profile.get('annotate', None))
            cache.set('ui_ann', tree)
    else:
        print 'From cache..'
    return tree

def download_yang(request, req):
    '''
    This API download yang schema from device
    '''
    logging.debug('Download Yang Schema')

    req = req.replace('<metadata>', '')
    req = req.replace('</metadata>', '')

    protocol, device, fmt, payload = Adapter.parse_request(req)
    if device.get('host', None) is None:
        return HttpResponse(Response.error('download', 'no host info'))

    session_dir = ServerSettings.schema_path(request.session.session_key)
    if not os.path.exists(session_dir):
        os.makedirs(session_dir)
    if not os.path.exists(session_dir):
        return HttpResponse(Response.error('download', 'No session directory'))

    for fname in os.listdir(session_dir):
        if fname.endswith('.yang'):
            fn = os.path.join(session_dir, fname)
            os.remove(fn)

    modules = ET.Element('modules')
    reqxml = ET.fromstring(req)
    schemas = reqxml.find('schemas')
    for sc in schemas:
        id = sc.text
        module = ET.Element('module')
        get_sc = ET.Element('get-schema')
        get_sc.set("xmlns", "urn:ietf:params:xml:ns:yang:ietf-netconf-monitoring")
        identifier = ET.Element("identifier")
        sfile = id.split('@')[0]
        identifier.text = sfile
        module.text = id+'.yang'
        get_sc.append(identifier)
        rpc.append(get_sc)
        modules.append(module)
        schema = Adapter.run_netconf(request.user.username, device, rpc)
        fname = os.path.join(session_dir, id+'.yang')
        with open(fname, 'w') as f:
            f.write(schema[0][0].text)
        rpc.remove(get_sc)

    return modules