def copytree(src, dst, symlinks=False):
	names = listdir(src)
	if os_path.isdir(dst):
		dst = os_path.join(dst, os_path.basename(src))
		if not os_path.isdir(dst):
			mkdir(dst)
	else:
		makedirs(dst)
	for name in names:
		srcname = os_path.join(src, name)
		dstname = os_path.join(dst, name)
		try:
			if symlinks and os_path.islink(srcname):
				linkto = readlink(srcname)
				symlink(linkto, dstname)
			elif os_path.isdir(srcname):
				copytree(srcname, dstname, symlinks)
			else:
				copyfile(srcname, dstname)
		except:
			print "dont copy srcname (no file or link or folder)"
	try:
		st = os_stat(src)
		mode = S_IMODE(st.st_mode)
		if have_chmod:
			chmod(dst, mode)
		if have_utime:
			utime(dst, (st.st_atime, st.st_mtime))
	except:
		print "copy stats for", src, "failed!"

def snapshot(source, destination, name=None):
    """Snapshot one directory to another. Specify names to snapshot small, named differences."""
    source = source + sep
    destination = destination + sep

    if not path.isdir(source):
        raise RuntimeError("source is not a directory")

    if path.exists(destination):
        if not path.isdir(destination):
            raise RuntimeError("destination is not a directory")

        if name is None:
            raise RuntimeError("can't snapshot base snapshot if destination exists")

    snapdir = path.join(destination, ".snapdir")
    
    if path.exists(path.join(source, ".snapdir")):
        raise RuntimeError("snapdir exists in source directory")

    if name is None:
        check_call(["rsync", "--del", "-av", source, destination])
        makedirs(snapdir)
    else:
        if not path.exists(snapdir):
            raise RuntimeError("No snapdir in destination directory")

        check_call(["rsync", "--del", "-av", "--only-write-batch={}".format(path.join(snapdir, name)), source, destination])

def get_repository_info(recipe_path):
    """This tries to get information about where a recipe came from.  This is different
    from the source - you can have a recipe in svn that gets source via git."""
    try:
        if exists(join(recipe_path, ".git")):
            origin = check_output_env(["git", "config", "--get", "remote.origin.url"],
                                      cwd=recipe_path)
            rev = check_output_env(["git", "rev-parse", "HEAD"], cwd=recipe_path)
            return "Origin {}, commit {}".format(origin, rev)
        elif isdir(join(recipe_path, ".hg")):
            origin = check_output_env(["hg", "paths", "default"], cwd=recipe_path)
            rev = check_output_env(["hg", "id"], cwd=recipe_path).split()[0]
            return "Origin {}, commit {}".format(origin, rev)
        elif isdir(join(recipe_path, ".svn")):
            info = check_output_env(["svn", "info"], cwd=recipe_path)
            server = re.search("Repository Root: (.*)$", info, flags=re.M).group(1)
            revision = re.search("Revision: (.*)$", info, flags=re.M).group(1)
            return "{}, Revision {}".format(server, revision)
        else:
            return "{}, last modified {}".format(recipe_path,
                                             time.ctime(os.path.getmtime(
                                                 join(recipe_path, "meta.yaml"))))
    except CalledProcessError:
        log.debug("Failed to checkout source in " + recipe_path)
        return "{}, last modified {}".format(recipe_path,
                                             time.ctime(os.path.getmtime(
                                                 join(recipe_path, "meta.yaml"))))

    def __scanDir( self, path ):
        """ Recursive function to scan one dir """
        # The path is with '/' at the end
        for item in os.listdir( path ):
            if self.shouldExclude( item ):
                continue

            # Exclude symlinks if they point to the other project
            # covered pieces
            candidate = path + item
            if islink( candidate ):
                realItem = realpath( candidate )
                if isdir( realItem ):
                    if self.isProjectDir( realItem ):
                        continue
                else:
                    if self.isProjectDir( os.path.dirname( realItem ) ):
                        continue

            if isdir( candidate ):
                self.filesList.add( candidate + sep )
                self.__scanDir( candidate + sep )
                continue
            self.filesList.add( candidate )
        return

		def iterLocations():
			if platform == 'android':
				# Under Android, the tcl set-up apparently differs from
				# other cross-platform setups. the search algorithm to find the
				# directory that will contain the tclConfig.sh script and the shared libs
				# is not applicable to Android. Instead, immediately return the correct
				# subdirectories to the routine that invokes iterLocations()
				sdl_android_port_path = environ['SDL_ANDROID_PORT_PATH']
				libpath = sdl_android_port_path + '/project/libs/armeabi'
				yield libpath
				tclpath = sdl_android_port_path + '/project/jni/tcl8.5/unix'
				yield tclpath
			else:
				if distroRoot is None or cls.isSystemLibrary(platform):
					if msysActive():
						roots = (msysPathToNative('/mingw32'), )
					else:
						roots = ('/usr/local', '/usr')
				else:
					roots = (distroRoot, )
				for root in roots:
					if isdir(root):
						for libdir in ('lib', 'lib64', 'lib/tcl'):
							libpath = root + '/' + libdir
							if isdir(libpath):
								yield libpath
								for entry in listdir(libpath):
									if entry.startswith('tcl8.'):
										tclpath = libpath + '/' + entry
										if isdir(tclpath):
											yield tclpath

        def __init__(self):

            # http://standards.freedesktop.org/basedir-spec/latest/ar01s03.html
            self.app_dir = join(getenv('XDG_DATA_HOME', expanduser('~/.local/share')), appname)
            if not isdir(self.app_dir):
                makedirs(self.app_dir)

            self.plugin_dir = join(self.app_dir, 'plugins')
            if not isdir(self.plugin_dir):
                mkdir(self.plugin_dir)

            self.home = expanduser('~')

            self.respath = dirname(__file__)

            self.filename = join(getenv('XDG_CONFIG_HOME', expanduser('~/.config')), appname, '%s.ini' % appname)
            if not isdir(dirname(self.filename)):
                makedirs(dirname(self.filename))

            self.config = RawConfigParser()
            try:
                self.config.readfp(codecs.open(self.filename, 'r', 'utf-8'))
            except:
                self.config.add_section('config')

            if not self.get('outdir') or not isdir(self.get('outdir')):
                self.set('outdir', expanduser('~'))

 def _retrieve_resource(self, uri):
     u"""
     Get the resource specified by the uri if it exist.
     Otherwise, raise a Exception
     """
     self._check_uri(uri)
     p = self._root + uri
     if isdir(p):
         body = ["<p>Directory Listing for "]
         body.append(uri)
         body.append("</p><ul>")
         dirs = []
         files = []
         for res in listdir(p):
             if isdir(p + res):
                 dirs.append(res + b'/')
             else:
                 files.append(res)
         dirs.sort()
         files.sort()
         resources = dirs + files
         for res in resources:
             body.append('<li><a href="{}">{}</a></li>'.format(res, res))
         body.append("</ul>")
         return ("".join(body), "text/html")
     elif isfile(p):
         with open(self._root + uri, 'rb') as resource:
             body = resource.read()
             content_type, content_encoding = mimetypes.guess_type(uri)
         return (body, content_type)
     else:
         raise ResourceNotFound

def download_file(url, name, root_destination='~/data/', zipfile=False,
                  replace=False):
    """Download a file from dropbox, google drive, or a URL.

    This will download a file and store it in a '~/data/` folder,
    creating directories if need be. It will also work for zip
    files, in which case it will unzip all of the files to the
    desired location.

    Parameters
    ----------
    url : string
        The url of the file to download. This may be a dropbox
        or google drive "share link", or a regular URL. If it
        is a share link, then it should point to a single file and
        not a folder. To download folders, zip them first.
    name : string
        The name / path of the file for the downloaded file, or
        the folder to zip the data into if the file is a zipfile.
    root_destination : string
        The root folder where data will be downloaded.
    zipfile : bool
        Whether the URL points to a zip file. If yes, it will be
        unzipped to root_destination + name.
    replace : bool
        If True and the URL points to a single file, overwrite the
        old file if possible.
    """
    # Make sure we have directories to dump files
    home = op.expanduser('~')
    tmpfile = home + '/tmp/tmp'
    if not op.isdir(home + '/data/'):
        print('Creating data folder...')
        os.makedirs(home + '/data/')

    if not op.isdir(home + '/tmp/'):
        print('Creating tmp folder...')
        os.makedirs(home + '/tmp/')

    download_path = _convert_url_to_downloadable(url)

    # Now save to the new destination
    out_path = root_destination.replace('~', home) + name
    if not op.isdir(op.dirname(out_path)):
        print('Creating path {} for output data'.format(out_path))
        os.makedirs(op.dirname(out_path))

    if zipfile is True:
        _fetch_file(download_path, tmpfile)
        myzip = ZipFile(tmpfile)
        myzip.extractall(out_path)
        os.remove(tmpfile)
    else:
        if len(name) == 0:
            raise ValueError('Cannot overwrite the root data directory')
        if replace is False and op.exists(out_path):
            raise ValueError('Path {} exists, use `replace=True` to '
                             'overwrite'.format(out_path))
        _fetch_file(download_path, out_path)
    print('Successfully moved file to {}'.format(out_path))

    def bootstrap_rustc_docs(self, force=False):
        self.ensure_bootstrapped()
        rust_root = self.config["tools"]["rust-root"]
        docs_dir = path.join(rust_root, "doc")
        if not force and path.exists(docs_dir):
            print("Rust docs already downloaded.", end=" ")
            print("Use |bootstrap-rust-docs --force| to download again.")
            return

        if path.isdir(docs_dir):
            shutil.rmtree(docs_dir)
        docs_name = self.rust_path().replace("rustc-", "rust-docs-")
        docs_url = ("https://static-rust-lang-org.s3.amazonaws.com/dist/rust-docs-nightly-%s.tar.gz"
                    % host_triple())
        tgz_file = path.join(rust_root, 'doc.tar.gz')

        download_file("Rust docs", docs_url, tgz_file)

        print("Extracting Rust docs...")
        temp_dir = path.join(rust_root, "temp_docs")
        if path.isdir(temp_dir):
            shutil.rmtree(temp_dir)
        extract(tgz_file, temp_dir)
        shutil.move(path.join(temp_dir, docs_name.split("/")[1],
                              "rust-docs", "share", "doc", "rust", "html"),
                    docs_dir)
        shutil.rmtree(temp_dir)
        print("Rust docs ready.")

    def test_censored_demo_files_are_deleted(self):
        """Demo files should be deleted when the demo is censored."""
        fout = StringIO()
        zf = zipfile.ZipFile(fout, "w")
        zf.writestr("demo.html", """<html></html""")
        zf.writestr("css/main.css", "h1 { color: red }")
        zf.writestr("js/main.js", 'alert("HELLO WORLD");')
        zf.close()

        s = Submission(
            title="Hello world", slug="hello-world", description="This is a hello world demo", creator=self.user
        )

        s.demo_package.save("play_demo.zip", ContentFile(fout.getvalue()))
        s.demo_package.close()
        s.clean()
        s.save()

        s.process_demo_package()

        path = s.demo_package.path.replace(".zip", "")

        ok_(isdir(path))
        ok_(isfile(s.demo_package.path))
        ok_(isfile("%s/index.html" % path))
        ok_(isfile("%s/css/main.css" % path))
        ok_(isfile("%s/js/main.js" % path))

        s.censor(url="http://example.com/censored-explanation")

        ok_(not isfile(s.demo_package.path))
        ok_(not isfile("%s/index.html" % path))
        ok_(not isfile("%s/css/main.css" % path))
        ok_(not isfile("%s/js/main.js" % path))
        ok_(not isdir(path))

    def test_demo_deletion(self):
        """Ensure that demo files are deleted along with submission record"""

        fout = StringIO()
        zf = zipfile.ZipFile(fout, "w")
        zf.writestr("demo.html", """<html></html""")
        zf.writestr("css/main.css", "h1 { color: red }")
        zf.writestr("js/main.js", 'alert("HELLO WORLD");')
        zf.close()

        s = Submission(
            title="Hello world", slug="hello-world", description="This is a hello world demo", creator=self.user
        )

        s.demo_package.save("play_demo.zip", ContentFile(fout.getvalue()))
        s.demo_package.close()
        s.clean()
        s.save()

        s.process_demo_package()

        path = s.demo_package.path.replace(".zip", "")

        ok_(isdir(path))
        ok_(isfile("%s/index.html" % path))
        ok_(isfile("%s/css/main.css" % path))
        ok_(isfile("%s/js/main.js" % path))

        s.delete()

        ok_(not isfile("%s/index.html" % path))
        ok_(not isfile("%s/css/main.css" % path))
        ok_(not isfile("%s/js/main.js" % path))
        ok_(not isdir(path))

  def __extract_queries_from_test_files(workload, query_names):
    """
    Enumerate all the query files for a workload and extract the query strings.
    If the user has specified a subset of queries to execute, only extract those query
    strings.
    """
    query_regex = None
    if query_names:
      # Build a single regex from all query name regex strings.
      query_regex = r'(?:' + '$)|('.join([name for name in query_names.split(',')]) + '$)'
    workload_base_dir = os.path.join(WORKLOAD_DIR, workload)
    if not isdir(workload_base_dir):
      raise ValueError,\
             "Workload '%s' not found at path '%s'" % (workload, workload_base_dir)

    query_dir = os.path.join(workload_base_dir, 'queries')
    if not isdir(query_dir):
      raise ValueError, "Workload query directory not found at path '%s'" % (query_dir)

    query_map = defaultdict(list)
    for query_file_name in WorkloadRunner.__enumerate_query_files(query_dir):
      LOG.debug('Parsing Query Test File: ' + query_file_name)
      sections = parse_query_test_file(query_file_name)
      test_name = re.sub('/', '.', query_file_name.split('.')[0])[1:]
      # If query_names is not none, only extract user specified queries to
      # the query map.
      if query_names:
        sections = [s for s in sections if re.match(query_regex, s['QUERY_NAME'], re.I)]
      for section in sections:
        query_map[test_name].append((section['QUERY_NAME'],
                                     (section['QUERY'], section['RESULTS'])))
    return query_map

def test_create_structure(tmpfolder):
    struct = {"my_file": "Some content",
              "my_folder": {
                  "my_dir_file": "Some other content",
                  "empty_file": "",
                  "file_not_created": None
              },
              "empty_folder": {}}
    expected = {"my_file": "Some content",
                "my_folder": {
                    "my_dir_file": "Some other content",
                    "empty_file": ""
                },
                "empty_folder": {}}
    changed, _ = structure.create_structure(struct, {})

    assert changed == expected
    assert isdir("my_folder")
    assert isdir("empty_folder")
    assert isfile("my_folder/my_dir_file")
    assert isfile("my_folder/empty_file")
    assert not isfile("my_folder/file_not_created")
    assert isfile("my_file")
    assert open("my_file").read() == "Some content"
    assert open("my_folder/my_dir_file").read() == "Some other content"
    assert open("my_folder/empty_file").read() == ""

def copy(src, dst, hardlink=False, keep_symlink=True):
    assert not P.isdir(src), 'Source path must not be a dir'
    assert not P.isdir(dst), 'Destination path must not be a dir'

    if keep_symlink and P.islink(src):
        assert not P.isabs(readlink(src)), 'Cannot copy symlink that points to an absolute path (%s)' % src
        logger.debug('%8s %s -> %s' % ('symlink', src, dst))
        if P.exists(dst):
            assert readlink(dst) == readlink(src), 'Refusing to retarget already-exported symlink %s' % dst
        else:
            symlink(readlink(src), dst)
        return

    if P.exists(dst):
        assert hash_file(src) == hash_file(dst), 'Refusing to overwrite already exported dst %s' % dst
    else:
        if hardlink:
            try:
                link(src, dst)
                logger.debug('%8s %s -> %s' % ('hardlink', src, dst))
                return
            except OSError, o:
                if o.errno != errno.EXDEV: # Invalid cross-device link, not an error, fall back to copy
                    raise

        logger.debug('%8s %s -> %s' % ('copy', src, dst))
        shutil.copy2(src, dst)

 def scanDir(self,dPath,usages):
     dName=path.basename(dPath)
     if dName[0]==".":
         return
     elif dName in ["lnInclude","Doxygen"]:
         return
     elif dName in ["Make","platform","bin"]:
         for f in listdir(dPath):
             if f[0]==".":
                 continue
             nPath=path.join(dPath,f)
             if path.isdir(nPath):
                 isBin=False
                 for end in ["Opt","Debug","Prof"]:
                     if f.find(end)>0 and (f.find(end)+len(end))==len(f):
                         isBin=True
                 if isBin:
                     sz=diskUsage(nPath)
                     try:
                         usages[f]+=sz
                     except KeyError:
                         usages[f]=sz
                        # print_("Found architecture",f,"in",dPath)
     else:
         try:
             for f in listdir(dPath):
                 nPath=path.join(dPath,f)
                 if path.isdir(nPath) and not path.islink(nPath):
                     self.scanDir(nPath,usages)
         except OSError:
             self.warning("Can't process",dPath)

def link(target, lnk, force=False):
    """
    Creates symbolic link 'lnk' pointing to 'target'.
    """

    if system() not in ('Linux', 'Windows', 'MSYS_NT-6.1'):
        print("{} operating system is not supported.".format(system()))
        return

    isdir = False

    lnk = path.normpath(path.expandvars(path.expanduser(lnk)))
    if path.isdir(target):
        isdir = True
    target = path.normpath(path.expandvars(path.expanduser(target)))

    if isdir:
        print("\n{} -> {} : DIR".format(lnk, target))
    else:
        print("\n{} -> {} : FILE".format(lnk, target))

    if path.isdir(lnk) or path.isfile(lnk):
        if not force:
            print("'{}': link exists".format(lnk))
            return
        else:
            remove(lnk)

    if system() in ('Linux', 'MSYS_NT-6.1'):
        Popen(['ln', '-s', target, lnk]).wait()
    elif system() == 'Windows':
        if isdir:
            CreateSymbolicLink(lnk, target, 1)
        else:
            CreateSymbolicLink(lnk, target, 0)

def check_artist_folder(path):
    """
    Check that the given path looks like an Artist folder should:

    - has a folder.jpg, possibly more than one image, but no music.
    - otherwise, only contains folders
    """

    if isdir(path):
        contents = os.listdir(path)
        files = filter(lambda x: isfile(join(path, x)) and not x.startswith('.'), contents)

        has_art = False
        has_folder_jpg = False
        only_contains_folders = True

        for item in files:
            item_path = join(path, item)
            if is_image_file(item_path):
                has_art = True
                if is_folder_art(item_path):
                    has_folder_jpg = True
            elif not (isdir(item_path) or is_ignored_file(item_path)):
                only_contains_folders = False

    return {
        'ok': has_art and only_contains_folders and has_folder_jpg,
        'has_art': has_art and has_folder_jpg,
        'only_contains_folders': only_contains_folders
    }

def compile_dir(env, src_path, dst_path, pattern=r'^[^\.].*\..*[^~]$',
                encoding='utf-8', base_dir=None,
                negative_pattern=r'^.*\.swp$'):
  """Compiles a directory of Jinja2 templates to python code.
  Params:
    `env`: a Jinja2 Environment instance.
    `src_path`: path to the source directory.
    `dst_path`: path to the destination directory.
    `encoding`: template encoding.
    `pattern`: a regular expression to match template file names.
    `base_dir`: the base path to be removed from the compiled template
      filename.
  """
  if base_dir is None:
    # In the first call, store the base dir.
    base_dir = src_path

  for filename in listdir(src_path):
    src_name = path.join(src_path, filename)
    dst_name = path.join(dst_path, filename)

    if path.isdir(src_name):
      if not path.isdir(dst_name):
        mkdir(dst_name)
      compile_dir(env, src_name, dst_name, encoding=encoding,
                  base_dir=base_dir)
    elif path.isfile(src_name) and re.match(pattern, filename) and \
          not re.match(negative_pattern, filename):
      compile_file(env, src_name, dst_name, encoding=encoding,
                   base_dir=base_dir)

def hg_source(source_dict, src_dir, hg_cache, verbose):
    ''' Download a source from Mercurial repo. '''
    if verbose:
        stdout = None
        stderr = None
    else:
        FNULL = open(os.devnull, 'w')
        stdout = FNULL
        stderr = FNULL

    hg_url = source_dict['hg_url']
    if not isdir(hg_cache):
        os.makedirs(hg_cache)
    hg_dn = hg_url.split(':')[-1].replace('/', '_')
    cache_repo = join(hg_cache, hg_dn)
    if isdir(cache_repo):
        check_call_env(['hg', 'pull'], cwd=cache_repo, stdout=stdout, stderr=stderr)
    else:
        check_call_env(['hg', 'clone', hg_url, cache_repo], stdout=stdout, stderr=stderr)
        assert isdir(cache_repo)

    # now clone in to work directory
    update = source_dict.get('hg_tag') or 'tip'
    if verbose:
        print('checkout: %r' % update)

    check_call_env(['hg', 'clone', cache_repo, src_dir], stdout=stdout,
                   stderr=stderr)
    check_call_env(['hg', 'update', '-C', update], cwd=src_dir, stdout=stdout,
                   stderr=stderr)

    if not verbose:
        FNULL.close()

    return src_dir

def _discover_subdatasets_recursively(
        discovered, top, trace, recursion_limit):
    # this beast walks the directory tree from a give `top` directory
    # and discovers valid repos that are scattered around, regardless
    # of whether they are already subdatasets or not
    # `trace` must be a list that has at least one element (the base
    # dataset)
    if recursion_limit is not None and len(trace) > recursion_limit:
        return
    if not isdir(top):
        return
    if not op.islink(top) and GitRepo.is_valid_repo(top):
        if top in discovered:
            # this was found already, assume everything beneath it too
            return
        discovered[top] = dict(
            path=top,
            # and its content
            process_content=True,
            type='dataset',
            parentds=trace[-1])
        # new node in the trace down
        trace = trace + [top]
    for path in listdir(top):
        path = opj(top, path)
        if not isdir(path):
            continue
        # next level down
        _discover_subdatasets_recursively(
            discovered, path, trace, recursion_limit)