def dealFile(path):

    dict = {}
    if not os.path.exists(path):
        return None
    for root, dirs, files in os.walk(path):

        for file in  files :

            if file.startswith(".DS_Store"):
                pass

            elif file.endswith("_base.xml"):

                list = countItem(path + "/"+file)
                try:
                    dict.update({'baseSize':getsize(path + "/"+ file)})
                except Exception, e:
                    dict.update({'baseSize': '0'})
                dict.update({"base_count": list[0]})
                dict.update({"base_tag_count": list[1]})

            elif file.endswith("_ext.xml"):
                list = countItem(path + "/"+file)
                try:
                    dict.update({'extSize':getsize(path + "/"+ file)})
                except Exception, e:
                    dict.update({'extSize':'0'})
                dict.update({"ext_count": list[0]})
                dict.update({"ext_tag_count": list[1]})

    def test_write_truncate(self):
        knowngood = {}
        filelist = list(self.pak2.listfiles())
        for f in filelist:
            self.pak2.open(f)
            data = self.pak2.read()
            knowngood[f] = [len(data), md5(data).hexdigest()]
        size = getsize(self.filename2)

        buf = "123456789"
        bufmd5 = md5(buf).hexdigest()
        for i in xrange(0, len(filelist), 2):
            self.pak2.open(filelist[i], "r")
            size -= len(self.pak2.read())
            self.pak2.close()
            self.pak2.open(filelist[i], "w")
            self.pak2.write(buf)
            self.pak2.close()
            size += len(buf)
            knowngood[filelist[i]][0] = len(buf)
            knowngood[filelist[i]][1] = bufmd5

        for f in filelist:
            self.assertEqual(filelist, list(self.pak2.listfiles()))
            self.pak2.open(f)
            data = self.pak2.read()
            self.assertEqual(len(data), knowngood[f][0])
            self.assertEqual(md5(data).hexdigest(), knowngood[f][1])

        del self.pak2
        self.assertEqual(getsize(self.filename2), size)

def compress(sources):
    """
        Compress a file/directory using 7zip.
        If the gain is not large enough, remove the archive, otherwise remove
        the source.
    """

    dest = "{0}.{1}".format(file_without_ext(sources[0]), extension)

    seven_zip(sources, dest)

    sources_size = 0
    for s in sources:
        sources_size += path.getsize(s)
        
    dest_size = path.getsize(dest)

    gain = sources_size - dest_size

    if gain < sources_size * min_ratio: # Not enought gain
        os.unlink(dest)
        return 0
    else:
        for s in sources:
            os.unlink(s)
        return gain

def comp (source_patch, target_patch):
	from os import path, walk
	from filecmp import cmp
	# выходное сообщение о найденных отличиях
	message = ''
	path_f = []
	tree = walk(source_patch)
	for d, dirs, files in tree:
		for f in files:
			patch = path.join(d,f) # формирование адреса
			path_f.append(patch)      # добавление адреса в список
		# перибираем адреса файлов из списка
		for patch in path_f:
			# выполняем сравнение файлов и в случае отличий получаем информацию о файлах
			# проверяем существование файла
			if not path.exists(patch.replace(source_patch, target_patch)):
				message = message + 'Отсутствует целевой файл: '+ patch.replace(source_patch, target_patch)
			# сверяем размеры исходного и целевого файла
			elif path.getsize(patch.replace(source_patch, target_patch)) <> path.getsize(patch):
				message = message + file_info(patch, patch.replace(source_patch, target_patch))
			# дата последней модификации
			elif path.getmtime(patch.replace(source_patch, target_patch)) <> path.getmtime(patch):
				message = message + file_info(patch, patch.replace(source_patch, target_patch))
			# сравниваем файлы
			elif not cmp(patch.replace(source_patch, target_patch), patch):
				message = message + file_info(patch, patch.replace(source_patch, target_patch))
		return message

def down_book(down_url,url,book_name,local_dir):
    """
    下载书籍，下载过程中后缀.tmp,下载完成判断大小，如果满足文件大小70%则认为成功并修改文件后缀
    """
    book_name = validatename(book_name)
    # book_name = book_name.replace("/","")
    down_dir_tmp = str(local_dir) + str(book_name)+ ".pdf"+".tmp"
    down_dir = str(local_dir) + str(book_name)+ ".pdf"
    if os.path.exists(down_dir) == True and abs(round(float(getsize(down_dir))/1024/1024,1) - round(float(size.replace(' MB',"")),1)) < 1: #判断书籍是否被下载过，如已存在文件则跳过
        #sys.exit()

        print ("....<"+book_name+"> already exists...")
        logging.info("....Books already exists....")
    elif os.path.exists(down_dir_tmp) == True or os.path.exists(down_dir) == False:
        if os.path.exists(down_dir_tmp) == True:
            print "...ReDownloading <"+book_name+">..."
            print "Original Size: "+size
            os.remove(down_dir_tmp)
        else:
            print "...Downloading <"+book_name+">..."
            print "Original Size: "+size
        rp = requests.get(down_url,headers = {'Referer':url},allow_redirects = False)
        r = requests.get(rp.headers['location'])
        with open(down_dir_tmp, "wb") as code:
           code.write(r.content)

        print "Actual Size: "+str(round(float(getsize(down_dir_tmp))/1024/1024,1))+" MB"
        if abs(round(float(getsize(down_dir_tmp))/1024/1024,1) 
            - round(float(size.replace(' MB',"")),1))/round(float(size.replace(' MB',"")),1) < 0.3:#此处可调整，如果下载不到原有的70%认为没下载成功
            os.rename(down_dir_tmp,down_dir)

 def command_patch_list_data(self):
   requested_version = (self.reported_version + 1)
   while requested_version in self.patch_catalog.getCatalog()[self.reported_client]:
     p_file = path.join(self.patch_catalog.getCatalog()['path'],
       self.patch_catalog.getCatalog()[self.reported_client][requested_version] + '.pat')
     p_head, p_tail = path.split(p_file)
     r_file = path.join(self.patch_catalog.getCatalog()['path'],
       self.patch_catalog.getCatalog()[self.reported_client][requested_version] + '.rtp')
     r_head, r_tail = path.split(r_file)
     file_listing = {'patname':p_tail.encode('ascii'),
                     'patnamelen':pack('>i', 
                       len(p_tail.encode('ascii'))),
                     'patlen':pack('>i', 
                       path.getsize(p_file)),
                     'rtpname':r_tail.encode('ascii'),
                     'rtpnamelen':pack('>i', 
                       len(r_tail.encode('ascii'))),
                     'rtplen':pack('>i', 
                       path.getsize(r_file))
                    }
     self.connection.send(file_listing['patnamelen'])
     self.connection.send(file_listing['patname'])
     self.connection.send(file_listing['patlen'])
     self.connection.send(file_listing['rtpnamelen'])
     self.connection.send(file_listing['rtpname'])
     self.connection.send(file_listing['rtplen'])
     print("\nSERVER << sent %s PatchListData entry %s\n" % (self.address, file_listing))
     requested_version = requested_version + 1
   self.connection.send(b'\x00\x00\x00\x00')
   return

def can_overwrite_old_file(old, new):
    """returns true if the old file does not exist and the new file is not significantly smaller"""
    if not PATH.exists(old):
        return True
    if not PATH.exists(new):
        return False
    return PATH.getsize(new) > PATH.getsize(old) * 0.7

    def repackage(self, extracted_apk_dir, dex_dir, have_locators):
        BaseDexMode.repackage(self, extracted_apk_dir, dex_dir, have_locators)

        metadata = DexMetadata(have_locators=have_locators,
                               store=self._store_id,
                               dependencies=self._dependencies)
        for i in range(1, 100):
            oldpath = join(dex_dir, self._dex_prefix + '%d.dex' % (i + 1))
            dexpath = join(dex_dir, self._store_name + '-%d.dex' % i)
            if not isfile(oldpath):
                break
            shutil.move(oldpath, dexpath)

            jarpath = dexpath + '.jar'
            create_dex_jar(jarpath, dexpath)
            metadata.add_dex(jarpath, BaseDexMode.get_canary(self, i))

            dex_meta_base = jarpath + '.meta'
            dex_meta_path = join(dex_dir, dex_meta_base)
            with open(dex_meta_path, 'w') as dex_meta:
                dex_meta.write('jar:%d dex:%d\n' %
                               (getsize(jarpath), getsize(dexpath)))

            shutil.move(dex_meta_path,
                        join(extracted_apk_dir, self._secondary_dir))
            shutil.move(jarpath, join(extracted_apk_dir,
                                      self._secondary_dir))
        jar_meta_path = join(dex_dir, 'metadata.txt')
        metadata.write(jar_meta_path)
        shutil.move(jar_meta_path, join(extracted_apk_dir, self._secondary_dir))

    def run(self, filename, pipedata):
        if 'optimize' in pipedata and not pipedata['optimize']:
            return
        self.bakery.logging_raw('### Optimize TTF {}'.format(filename))
        # copied from https://code.google.com/p/noto/source/browse/nototools/subset.py
        from fontTools.subset import Options, Subsetter, load_font, save_font

        options = Options()
        options.layout_features = "*"
        options.name_IDs = "*"
        options.hinting = True
        options.notdef_outline = True

        font = load_font(op.join(self.builddir, filename), options)
        subsetter = Subsetter(options=options)
        subsetter.populate(glyphs=font.getGlyphOrder())
        subsetter.subset(font)
        save_font(font, op.join(self.builddir, filename + '.opt'), options)

        newsize = op.getsize(op.join(self.builddir, filename + '.opt'))
        origsize = op.getsize(op.join(self.builddir, filename))

        # compare filesizes TODO print analysis of this :)
        comment = "# look at the size savings of that subset process"
        self.bakery.logging_cmd("ls -l '%s'* %s" % (filename, comment))

        statusmessage = "{0}.opt: {1} bytes\n{0}: {2} bytes\n"
        self.bakery.logging_raw(statusmessage.format(filename, newsize, origsize))

        # move ttx files to src
        shutil.move(op.join(self.builddir, filename + '.opt'),
                    op.join(self.builddir, filename),
                    log=self.bakery.logger)

def build_project(project, _zip, _url, _alias, _replace, _create, _option):
  """Build project."""
  if _option:
    project.properties = flatten(project.properties)
    # to make sure we properly override nested options, we flatten first
    project.properties.update(_parse_option(_option))
  if _zip:
    if osp.isdir(_zip):
      _zip = osp.join(_zip, '%s.zip' % (project.versioned_name, ))
    project.build(_zip, overwrite=_replace)
    sys.stdout.write(
      'Project %s successfully built and saved as %r (size: %s).\n'
      % (project, _zip, human_readable(osp.getsize(_zip)))
    )
  else:
    with temppath() as _zip:
      project.build(_zip)
      archive_name = '%s.zip' % (project.versioned_name, )
      session = _get_session(_url, _alias)
      res = _upload_zip(session, project.name, _zip, _create, archive_name)
      sys.stdout.write(
        'Project %s successfully built and uploaded '
        '(id: %s, size: %s, upload: %s).\n'
        'Details at %s/manager?project=%s\n'
        % (
          project,
          res['projectId'],
          human_readable(osp.getsize(_zip)),
          res['version'],
          session.url,
          project,
        )
      )

def build_project(project, zip, url, alias, replace, create):
  """Build project."""
  if zip:
    project.build(zip, overwrite=replace)
    stdout.write(
      'Project successfully built and saved as %r (size: %s).\n'
      % (zip, human_readable(getsize(zip)))
    )
  else:
    with temppath() as zip:
      project.build(zip)
      session = Session(url, alias)
      while True:
        try:
          res = session.upload_project(project.name, zip)
        except AzkabanError as err:
          if create:
            session.create_project(project.name, project.name)
          else:
            raise err
        else:
          break
      stdout.write(
        'Project %s successfully built and uploaded '
        '(id: %s, size: %s, version: %s).\n'
        'Details at %s/manager?project=%s\n'
        % (
          project,
          res['projectId'],
          human_readable(getsize(zip)),
          res['version'],
          session.url,
          project,
        )
      )

    def compressFile(self,fName):
        if self.verbose>1:
            print_("  Compressing",fName)
        zippedName=fName+".gz"
        if path.exists(zippedName):
            self.warning("Zipped file",zippedName,"already existing for",fName)
            return
        oldSize=path.getsize(fName)
        if oldSize<self.bigSize:
            if self.verbose>2:
                print_("   Skipping because it is too small")
            self.nrSkipped+=1
            return

        # use gzip because that way the responsibility of removing the old file is with a 'tried and testd' program
        ret=subprocess.call(["gzip",fName])
        if ret!=0 or not path.exists(zippedName) or path.exists(fName):
            self.warning("Problem compressing file",fName)
            self.nrProblems+=1
            return
        newSize=path.getsize(zippedName)
        if newSize>oldSize:
            self.warning("Compression of",fName,"increased the filesize. Old:",
                         humanReadableSize(oldSize),"New:",humanReadableSize(newSize))

        if self.verbose>2:
            print_("   Old size:",humanReadableSize(oldSize),"New size:",humanReadableSize(newSize))

        self.nrFiles+=1
        self.prevSize+=oldSize
        self.nowSize+=newSize

def _calcsize(file_to_torrent):
    if not isdir(file_to_torrent):
        return getsize(file_to_torrent)
    total = 0
    for s in _subfiles(abspath(file_to_torrent)):
        total += getsize(s[1])
    return total

def compare_dns_dirs():
	print(' Checking if there is need for an update .... ')
	
	#first check to see if .ip_tmp_path exists
	if ( path.exists('.ip_tmp_path') and (path.isdir('.ip_tmp_path')) ):
		print(' Give me just a few seconds more')
		sleep(2)
		
		if ( int(path.getsize('.ip_tmp')) <= int(path.getsize('.ip_tmp_path')) ):
			print(' \n Looks like new content is available ')
			
			# copying new content in .dns_tmp_path to .dns_tmp
			try:
				rmtree('.dns_tmp')
				copytree('.dns_tmp_path','.dns_tmp')
			except:
				print(' Failed to copy new data ... ')
				print(' Exiting ... ')
				exit(0)
			else:
				print(' Successfully moved new data')				
		else:
			print(' Nothing new was added ... ')
			print(' Exiting ... ')
			exit(0)
	else:
		print(' This is first run ... \n moving on ... ')
		sleep(2)

def duplicate(original, new, other):

	if not exists(new):
		return False

	if new == original:
		del_file(original)
		return True

	try:
		if DadVision.args.force_rename or (other and other.lower() in ["proper"]):
			log.warn("Replacing Existing with repack or Force Rename Requested: {}".format(original))
			del_file(new)
			return False

		if getsize(original) > getsize(new):
			log.info("Replacing Existing with Larger Version: {}".format(new))
			del_file(new)
			return False

		log.trace("Comparing existing file to new, may run for some time.")
		if filecmp.cmp(original, new):
			log.info("Deleting New File, Same File already at destination!: {}".format(new))
			del_file(original)
			return True
		del_file(new)
		return False
	except (InvalidPath, OSError), e:
		log.error("Unable to remove File: {}".format(e))
		return True

    def disabled_test_xml_quiet(self):
        """ Tests the 'quiet' parameter of the MARC8ToUnicode class,
            passed in via the pymarc.record_to_xml() method
        """
        outfile = 'test/dummy_stderr.txt'
        # truncate outfile in case someone's fiddled with it
        open(outfile, 'wb').close()
        # redirect stderr
        sys.stderr = open(outfile, 'wb')
        # reload pymarc so it picks up the new sys.stderr
        reload(pymarc)
        # get problematic record
        record = next(pymarc.reader.MARCReader(open('test/utf8_errors.dat', 'rb')))
        # record_to_xml() with quiet set to False should generate errors
        #   and write them to sys.stderr
        xml = pymarc.record_to_xml(record, quiet=False)
        # close dummy stderr so we can accurately get its size
        sys.stderr.close()
        # file size should be greater than 0
        self.assertNotEqual(getsize(outfile), 0)

        # truncate file again
        open(outfile, 'wb').close()
        # be sure its truncated
        self.assertEqual(getsize(outfile), 0)
        # redirect stderr again
        sys.stderr = open(outfile, 'wb')
        reload(pymarc)
        # record_to_xml() with quiet set to True should not generate errors
        xml = pymarc.record_to_xml(record, quiet=True)
        # close dummy stderr
        sys.stderr.close()
        # no errors should have been written
        self.assertEqual(getsize(outfile), 0)

    def processfile(self, filename):
        """Renames the specified image to a backup path,
        and writes out the image again with optimal settings."""
        try:
            # Skip read-only files
            if not stat(filename)[0] & S_IWRITE:
                print 'Ignoring read-only file "' + filename + '".'
                return False

            # Create a backup
            backupname = filename + "." + self.backupextension

            if isfile(backupname):
                print 'Ignoring file "' + filename + '" for which existing backup file is present.'
                return False

            rename(filename, backupname)
        except Exception as e:
            stderr.write('Skipping file "' + filename + '" for which backup cannot be made: ' + str(e) + "\n")
            return False

        ok = False

        try:
            # Open the image
            with open(backupname, "rb") as file:
                img = Image.open(file)

                # Check that it's a supported format
                format = str(img.format)
                if format != "PNG" and format != "JPEG":
                    print 'Ignoring file "' + filename + '" with unsupported format ' + format
                    return False

                # This line avoids problems that can arise saving larger JPEG files with PIL
                ImageFile.MAXBLOCK = img.size[0] * img.size[1]

                # The 'quality' option is ignored for PNG files
                img.save(filename, quality=90, optimize=True)

            # Check that we've actually made it smaller
            origsize = getsize(backupname)
            newsize = getsize(filename)

            if newsize >= origsize:
                print 'Cannot further compress "' + filename + '".'
                return False

            # Successful compression
            ok = True
        except Exception as e:
            stderr.write('Failure whilst processing "' + filename + '": ' + str(e) + "\n")
        finally:
            if not ok:
                try:
                    move(backupname, filename)
                except Exception as e:
                    stderr.write('ERROR: could not restore backup file for "' + filename + '": ' + str(e) + "\n")

        return ok

def unpack_file_from_file(in_file, out_file):
    """ Uncompress a file from a file.

    Parameters
    ----------
    in_file : str
        the name of the input file
    out_file : str
        the name of the output file

    Returns
    -------
    metadata : bytes
        the metadata contained in the file if present

    Raises
    ------

    FormatVersionMismatch
        if the file has an unmatching format version number
    ChecksumMismatch
        if any of the chunks fail to produce the correct checksum
    """
    in_file_size = path.getsize(in_file)
    log.verbose('input file size: %s' % pretty_size(in_file_size))
    with open(in_file, 'rb') as input_fp, open(out_file, 'wb') as output_fp:
        source = CompressedFPSource(input_fp)
        sink = PlainFPSink(output_fp, source.nchunks)
        unpack(source, sink)
    out_file_size = path.getsize(out_file)
    log.verbose('output file size: %s' % pretty_size(out_file_size))
    log.verbose('decompression ratio: %f' % (out_file_size / in_file_size))
    return source.metadata

def getchunks(infile, n_cpus, scheduler = 'guided'):
    # Divide input data based on scheduler type
    if scheduler == 'static':
        size = getsize(infile) / n_cpus
    else:
        size = getsize(infile) / (n_cpus * 20)

    # Open input file    
    try:    
        ifile = open(infile)
    except:
        print >> stderr, 'Error: Not able to open ', infile, '. Exiting.'
        exit(-1)

    # Create chunk of data to be distributed to nodes    
    while 1:
        start = ifile.tell()
        ifile.seek(size, 1)
        s = ifile.readline()
        yield start, ifile.tell() - start
        if not s:
            break

    # Close the input file    
    try:    
        ifile.close()
    except:
        print >> stderr, 'Warning: Error closing the file ', ifile

def dir_cache_data(path):
    """
    Return the data to store in the cache for directory at *path*.
    """
    path = force_utf8(path)
    files = []
    directories = []
    for entry in os.listdir(path):
        for pattern in settings.EXCLUDE_FILES:
            if pattern.match(entry):
                continue
        entry_path = op.join(path, entry)
        if not op.exists(entry_path):
            # File was deleted during directory listing
            continue
        timestamp = op.getmtime(entry_path)
        if op.isdir(entry_path):
            size = 0
            for dirpath, dirnames, filenames in os.walk(entry_path):
                for f in filenames:
                    fp = op.join(dirpath, f)
                    if op.exists(fp):
                        size += op.getsize(fp)
            directories.append((entry, size, timestamp))
        else:
            size = op.getsize(entry_path)
            files.append((entry, size, timestamp))
    return directories, files