def test_sorting(self):
        # Ensure the header is sorted to the first entry so that its charset
        # can be applied to all subsequent messages by GNUTranslations
        # (ensuring all messages are safely converted to unicode)
        catalog = Catalog(locale="en_US")
        catalog.add(
            u"",
            """\
"Content-Type: text/plain; charset=utf-8\n"
"Content-Transfer-Encoding: 8bit\n""",
        )
        catalog.add(u"foo", "Voh")
        catalog.add((u"There is", u"There are"), (u"Es gibt", u"Es gibt"))
        catalog.add(u"Fizz", "")
        catalog.add(("Fuzz", "Fuzzes"), ("", ""))
        buf = StringIO()
        mofile.write_mo(buf, catalog)
        buf.seek(0)
        translations = gettext.GNUTranslations(fp=buf)
        self.assertEqual(u"Voh", translations.ugettext("foo"))
        assert isinstance(translations.ugettext("foo"), unicode)
        self.assertEqual(u"Es gibt", translations.ungettext("There is", "There are", 1))
        assert isinstance(translations.ungettext("There is", "There are", 1), unicode)
        self.assertEqual(u"Fizz", translations.ugettext("Fizz"))
        assert isinstance(translations.ugettext("Fizz"), unicode)
        self.assertEqual(u"Fuzz", translations.ugettext("Fuzz"))
        assert isinstance(translations.ugettext("Fuzz"), unicode)
        self.assertEqual(u"Fuzzes", translations.ugettext("Fuzzes"))
        assert isinstance(translations.ugettext("Fuzzes"), unicode)

 def setUp(self):
     # Use a locale which won't fail to run the tests
     os.environ["LANG"] = "en_US.UTF-8"
     messages1 = [
         ("foo", {"string": "Voh"}),
         ("foo", {"string": "VohCTX", "context": "foo"}),
         (("foo1", "foos1"), {"string": ("Voh1", "Vohs1")}),
         (("foo1", "foos1"), {"string": ("VohCTX1", "VohsCTX1"), "context": "foo"}),
     ]
     messages2 = [
         ("foo", {"string": "VohD"}),
         ("foo", {"string": "VohCTXD", "context": "foo"}),
         (("foo1", "foos1"), {"string": ("VohD1", "VohsD1")}),
         (("foo1", "foos1"), {"string": ("VohCTXD1", "VohsCTXD1"), "context": "foo"}),
     ]
     catalog1 = Catalog(locale="en_GB", domain="messages")
     catalog2 = Catalog(locale="en_GB", domain="messages1")
     for ids, kwargs in messages1:
         catalog1.add(ids, **kwargs)
     for ids, kwargs in messages2:
         catalog2.add(ids, **kwargs)
     catalog1_fp = BytesIO()
     catalog2_fp = BytesIO()
     write_mo(catalog1_fp, catalog1)
     catalog1_fp.seek(0)
     write_mo(catalog2_fp, catalog2)
     catalog2_fp.seek(0)
     translations1 = support.Translations(catalog1_fp)
     translations2 = support.Translations(catalog2_fp, domain="messages1")
     self.translations = translations1.add(translations2, merge=False)

 def compile(self):
     self.pod.catalogs.clear_gettext_cache()
     localization = self.pod.podspec.localization
     if localization is None:
         return
     compile_fuzzy = localization.get('compile_fuzzy')
     mo_filename = self.mo_path
     num_translated = 0
     num_total = 0
     for message in list(self)[1:]:
         if message.string:
             num_translated += 1
         num_total += 1
     try:
         for message, errors in self.check():
             for error in errors:
                 text = 'Error compiling ({}:{}): {}'
                 message = text.format(self.locale, message.lineno, error)
                 self.pod.logger.error(message)
     except IOError:
         self.pod.logger.info('Skipped catalog check for: {}'.format(self))
     text = 'Compiled: {} ({}/{})'
     self.pod.logger.info(text.format(self.locale, num_translated, num_total))
     mo_file = self.pod.open_file(mo_filename, 'w')
     try:
         mofile.write_mo(mo_file, self, use_fuzzy=compile_fuzzy)
     finally:
         mo_file.close()

 def get_po_catalog(self):
     cat = StringIO()
     with closing(zipopen(self.file_base + '.po')) as f:
         po = read_po(f)
         write_mo(cat, po)
         cat.seek(0)
         return cat

    def compile_catalogs(self, languages=None):
        """
        Compile all knowed catalogs
        """
        languages = languages or self.settings.LANGUAGES
        for locale in languages:
            catalog_path = self.get_catalog_path(locale)
            self.logger.info('Compiling catalog (PO) for language {0} at {1}'.format(locale, catalog_path))
            infile = open(catalog_path, 'r')
            try:
                catalog = read_po(infile, locale)
            finally:
                infile.close()
            
            # Check errors in catalog
            errs = False
            for message, errors in catalog.check():
                for error in errors:
                    errs = True
                    self.logger.warning('Error at line {0}: {1}'.format(message.lineno, error))
            # Don't overwrite the previous MO file if there have been error
            if errs:
                self.logger.error('There has been errors within the catalog, compilation has been aborted')
                break

            outfile = open(self.get_compiled_catalog_path(locale), 'wb')
            try:
                write_mo(outfile, catalog, use_fuzzy=False)
            finally:
                outfile.close()

def compile_domain(domain):
    for locale in gettext_finder.GETTEXT_LANGUAGES:
        popath = os.path.join('locale', locale, "LC_MESSAGES", domain + ".po")
        mopath = os.path.join('locale', locale, "LC_MESSAGES", domain + ".mo")

        with open(mopath, 'w') as mo_f, open(popath) as po_f:
            write_mo(mo_f, read_po(po_f))

def test_translation_plugins(app, tmpdir):
    session.lang = 'fr_FR'
    plugin = MockPlugin(plugin_engine, app)
    app.extensions['pluginengine'].plugins['dummy'] = plugin
    plugin.root_path = tmpdir.strpath
    french_core_str = DICTIONARIES['fr_FR']['This is not a string']
    french_plugin_str = "This is not le french string"

    trans_dir = os.path.join(plugin.root_path, 'translations', 'fr_FR', 'LC_MESSAGES')
    os.makedirs(trans_dir)

    # Create proper *.mo file for plugin translation
    with open(os.path.join(trans_dir, 'messages.mo'), 'wb') as f:
        catalog = Catalog(locale='fr_FR', domain='plugin')
        catalog.add("This is not a string", "This is not le french string")
        write_mo(f, catalog)

    gettext_plugin = make_bound_gettext('dummy')

    assert _(u'This is not a string') == french_core_str
    assert gettext_context(u"This is not a string") == french_core_str
    assert gettext_plugin(u"This is not a string") == french_plugin_str

    with plugin.plugin_context():
        assert _(u'This is not a string') == french_core_str
        assert gettext_context(u"This is not a string") == french_plugin_str
        assert gettext_plugin(u"This is not a string") == french_plugin_str

    def run(self, root):

        i18n_dir = self.extension.getConfig('i18n_dir')
        pot_path = os.path.join(i18n_dir, 'messages.pot')

        if os.path.exists(pot_path):
            with open(pot_path, 'r') as f:
                catalog = pofile.read_po(f)
        else:
            catalog = Catalog()

        lang = self.extension.getConfig('i18n_lang')
        mo_path = os.path.join(i18n_dir, lang, 'LC_MESSAGES', 'messages.mo')
        po_path = os.path.join(i18n_dir, lang, 'LC_MESSAGES', 'messages.po')

        if os.path.exists(po_path):
            with open(po_path, 'r') as f:
                lang_catalog = pofile.read_po(f)
            with open(mo_path, 'w') as mo:
                mofile.write_mo(mo, lang_catalog)

        translations = Translations.load(i18n_dir, locales=[lang])
        self.translate(catalog, translations, root)

        with open(pot_path, 'w') as pot_file:
            pofile.write_po(pot_file, catalog)

  def compile(self, use_fuzzy=False):
    mo_dirpath = os.path.dirname(self.pod_path)
    mo_filename = os.path.join(mo_dirpath, 'messages.mo')

    num_translated = 0
    num_total = 0
    for message in list(self)[1:]:
      if message.string:
        num_translated += 1
      num_total += 1

    try:
      for message, errors in self.check():
        for error in errors:
          logging.error('Error: {}:{}: {}'.format(self.locale, message.lineno, error))
    except IOError:
      logging.info('Skipped catalog check for: {}'.format(self))

    text = 'Compiled: {} ({}/{})'
    self.pod.logger.info(text.format(self.locale, num_translated, num_total))

    mo_file = self.pod.open_file(mo_filename, 'w')
    try:
      mofile.write_mo(mo_file, self, use_fuzzy=use_fuzzy)
    finally:
      mo_file.close()

 def generate_mo (self, file_name, use_fuzzy = False) :
     self._make_dir (file_name)
     file = open    (file_name, 'wb')
     try:
         write_mo   (file, self.catalog, use_fuzzy = use_fuzzy)
     finally:
         file.close ()

    def test_sorting(self):
        # Ensure the header is sorted to the first entry so that its charset
        # can be applied to all subsequent messages by GNUTranslations
        # (ensuring all messages are safely converted to unicode)
        catalog = Catalog(locale='en_US')
        catalog.add(u'', '''\
"Content-Type: text/plain; charset=utf-8\n"
"Content-Transfer-Encoding: 8bit\n''')
        catalog.add(u'foo', 'Voh')
        catalog.add((u'There is', u'There are'), (u'Es gibt', u'Es gibt'))
        catalog.add(u'Fizz', '')
        catalog.add(('Fuzz', 'Fuzzes'), ('', ''))
        buf = StringIO()
        mofile.write_mo(buf, catalog)
        buf.seek(0)
        translations = gettext.GNUTranslations(fp=buf)
        self.assertEqual(u'Voh', translations.ugettext('foo'))
        assert isinstance(translations.ugettext('foo'), unicode)
        self.assertEqual(u'Es gibt', translations.ungettext('There is', 'There are', 1))
        assert isinstance(translations.ungettext('There is', 'There are', 1), unicode)
        self.assertEqual(u'Fizz', translations.ugettext('Fizz'))
        assert isinstance(translations.ugettext('Fizz'), unicode)
        self.assertEqual(u'Fuzz', translations.ugettext('Fuzz'))
        assert isinstance(translations.ugettext('Fuzz'), unicode)
        self.assertEqual(u'Fuzzes', translations.ugettext('Fuzzes'))
        assert isinstance(translations.ugettext('Fuzzes'), unicode)

  def recompile_mo(self, use_fuzzy=False):
    locale = str(self.locale)
    po_filename = os.path.join(self.path, 'LC_MESSAGES', 'messages.po')
    mo_filename = os.path.join(self.path, 'LC_MESSAGES', 'messages.mo')
    po_file = self.pod.open_file(po_filename)
    try:
      catalog = pofile.read_po(po_file, locale)
    finally:
      po_file.close()

    num_translated = 0
    num_total = 0
    for message in list(catalog)[1:]:
      if message.string:
        num_translated += 1
      num_total += 1

    if catalog.fuzzy and not use_fuzzy:
      logging.info('Catalog {} is marked as fuzzy, skipping.'.format(po_filename))

    try:
      for message, errors in catalog.check():
        for error in errors:
          logging.error('Error: {}:{}: {}'.format(po_filename, message.lineno, error))
    except IOError:
      logging.info('Skipped catalog check.')

    text = 'Compiling {}/{} translated strings to {}'
    logging.info(text.format(num_translated, num_total, mo_filename))

    mo_file = self.pod.open_file(mo_filename, 'w')
    try:
      mofile.write_mo(mo_file, catalog, use_fuzzy=use_fuzzy)
    finally:
      mo_file.close()

    def test_headers(self):
        for x in range(1, 7):
            text = "{0} This is a h{1}".format('#' * x, x)
            expected = '<h{0} id="esto-es-un-h{0}">Esto es un h{0}</h{0}>'.format(x)

            with TempDir() as d:

                c = catalog.Catalog(locale='es_ES')
                c.add('This is a h{0}'.format(x), 'Esto es un h{0}'.format(x))
                os.mkdir(os.path.join(d.dir, 'es_ES'))
                lc_messages = os.path.join(d.dir, 'es_ES', 'LC_MESSAGES')
                os.mkdir(lc_messages)
                mo_file = os.path.join(lc_messages, 'messages.mo')
                with open(mo_file, 'w') as f:
                    mofile.write_mo(f, c)

                result = self.markdown(
                    text,
                    extensions=['markdown.extensions.toc'],
                    extension_configs={
                        'markdown_i18n': {
                            'i18n_dir': d.dir,
                            'i18n_lang': 'es_ES'
                        }
                    }
                )
                self.assertEqual(expected, result)

    def _run_domain(self, domain):
        po_files = []
        mo_files = []

        if not self.input_file:
            if self.locale:
                po_files.append((self.locale, os.path.join(self.directory, self.locale, "LC_MESSAGES", domain + ".po")))
                mo_files.append(os.path.join(self.directory, self.locale, "LC_MESSAGES", domain + ".mo"))
            else:
                for locale in os.listdir(self.directory):
                    po_file = os.path.join(self.directory, locale, "LC_MESSAGES", domain + ".po")
                    if os.path.exists(po_file):
                        po_files.append((locale, po_file))
                        mo_files.append(os.path.join(self.directory, locale, "LC_MESSAGES", domain + ".mo"))
        else:
            po_files.append((self.locale, self.input_file))
            if self.output_file:
                mo_files.append(self.output_file)
            else:
                mo_files.append(os.path.join(self.directory, self.locale, "LC_MESSAGES", domain + ".mo"))

        if not po_files:
            raise DistutilsOptionError("no message catalogs found")

        for idx, (locale, po_file) in enumerate(po_files):
            mo_file = mo_files[idx]
            infile = open(po_file, "rb")
            try:
                catalog = read_po(infile, locale)
            finally:
                infile.close()

            if self.statistics:
                translated = 0
                for message in list(catalog)[1:]:
                    if message.string:
                        translated += 1
                percentage = 0
                if len(catalog):
                    percentage = translated * 100 // len(catalog)
                self.log.info(
                    "%d of %d messages (%d%%) translated in %r", translated, len(catalog), percentage, po_file
                )

            if catalog.fuzzy and not self.use_fuzzy:
                self.log.info("catalog %r is marked as fuzzy, skipping", po_file)
                continue

            for message, errors in catalog.check():
                for error in errors:
                    self.log.error("error: %s:%d: %s", po_file, message.lineno, error)

            self.log.info("compiling catalog %r to %r", po_file, mo_file)

            outfile = open(mo_file, "wb")
            try:
                write_mo(outfile, catalog, use_fuzzy=self.use_fuzzy)
            finally:
                outfile.close()

def catalog_to_translations(catalog):
    """
    Helper function which converts catalog object to translation
    """
    buf = BytesIO()
    write_mo(buf, catalog, use_fuzzy=True)
    buf.seek(0)
    return Translations(fp=buf)

def create_i18n_files(lang, quiet=True):
    sql = """
        SELECT * FROM translation
        WHERE lang = ? and active=1;
    """

    result = conn.execute(sql, lang)

    catalog = Catalog(locale=lang)

    for row in result:
        if row.plural:
            key = (row.id, row.plural)
            values = [
                row.trans1,
                row.trans2,
                row.trans3,
                row.trans4,
            ]
            value = []
            for v in values:
                if v:
                    value.append(v)
            if len(value) == 1:
                value = value[0]
            else:
                value = tuple(value)

        else:
            key = row.id
            value = row.trans1
        catalog.add(key, value)

    path = os.path.join(DIR, 'translations', lang, 'LC_MESSAGES')
    try:
        os.makedirs(path)
    except OSError:
        pass

    outfile = open(os.path.join(path, 'messages.po'), 'wb')
    try:
        if not quiet:
            print 'writing PO for', lang
        write_po(outfile, catalog)
    finally:
        outfile.close()

    outfile = open(os.path.join(path, 'messages.mo'), 'wb')
    try:
        if not quiet:
            print 'writing MO for', lang
        write_mo(outfile, catalog)
    finally:
        outfile.close()

 def inject_translations(self, locale, content):
     po_file_to_merge = cStringIO.StringIO()
     po_file_to_merge.write(content)
     po_file_to_merge.seek(0)
     catalog_to_merge = pofile.read_po(po_file_to_merge, locale)
     mo_fp = cStringIO.StringIO()
     mofile.write_mo(mo_fp, catalog_to_merge)
     mo_fp.seek(0)
     translation_obj = support.Translations(mo_fp, domain='messages')
     self._gettext_translations[locale] = translation_obj
     self.pod.logger.info('Injected translations -> {}'.format(locale))

def write_mo(filename, catalog):
    """write mo file from catalog object

    :param unicode filename: path to mo file
    :param catalog: catalog object
    :return: None
    """
    dirname = os.path.dirname(filename)
    if not os.path.exists(dirname):
        os.makedirs(dirname)
    with io.open(filename, 'wb') as f:
        mofile.write_mo(f, catalog)

def po_project_export(project, project_version, archive_fileobj, compile_mo=True):
    """
    Export all catalogs from a project into PO files with the good directory 
    structure
    
    NOTE: One improvement would be to not perform two querysets for each catalog (one for PO, one 'solid' for MO)
    """
    archive_files = []
    
    # Template catalog POT file
    template_file = StringIO()
    write_po(template_file, project_version.get_babel_template(), sort_by_file=False, ignore_obsolete=True, include_previous=False)
    template_file.seek(0)
    archive_files.append( (settings.POT_ARCHIVE_PATH.format(catalog_filename=project.domain), template_file) )
    
    # Catalog PO files
    for catalog in project_version.catalog_set.all():
        po_file_path = settings.PO_ARCHIVE_PATH.format(locale=catalog.locale, catalog_filename=project.domain)
        mo_file_path = settings.MO_ARCHIVE_PATH.format(locale=catalog.locale, catalog_filename=project.domain)
        
        # Write the PO to a buffer string
        po_file = StringIO()
        write_po(po_file, catalog.get_babel_catalog(), sort_by_file=False, ignore_obsolete=True, include_previous=False)
        po_file.seek(0)
        
        # Append the PO file to the archive manifest
        archive_files.append( (po_file_path, po_file) )
        
        # Write the MO to a buffer string
        mo_file = StringIO()
        write_mo(mo_file, catalog.get_babel_catalog(solid=True), use_fuzzy=False)
        mo_file.seek(0)
        
        # Append the MO file to the archive manifest
        archive_files.append( (mo_file_path, mo_file) )
        
    
    # Open and fill tarball archive
    archive = tarfile.open("{0}.tar.gz".format(project.slug), mode="w:gz", fileobj=archive_fileobj)
    mtime = time.time()
    
    # Build the tarball from the manifest
    for name,content in archive_files:
        info = tarfile.TarInfo(name)
        info.size=len(content.getvalue())
        info.mtime = mtime
        archive.addfile(tarinfo=info, fileobj=content)
        #
        content.close()
    
    archive.close()
    
    return archive_fileobj

 def test_load(self):
     tempdir = tempfile.mkdtemp()
     try:
         messages_dir = os.path.join(tempdir, 'fr', 'LC_MESSAGES')
         os.makedirs(messages_dir)
         catalog = Catalog(locale='fr', domain='messages')
         catalog.add('foo', 'bar')
         write_mo(file(os.path.join(messages_dir, 'messages.mo'), 'wb'), catalog)
         
         translations = support.Translations.load(tempdir, locales=('fr',), domain='messages')
         self.assertEqual('bar', translations.gettext('foo'))
     finally:
         shutil.rmtree(tempdir)