def test_converter_opensubtitles(self):
        self.assertEqual(Language('fra').opensubtitles, Language('fra').alpha3b)
        self.assertEqual(Language('por', 'BR').opensubtitles, 'pob')
        self.assertEqual(Language.fromopensubtitles('fre'), Language('fra'))
        self.assertEqual(Language.fromopensubtitles('pob'), Language('por', 'BR'))
        self.assertEqual(Language.fromopensubtitles('pb'), Language('por', 'BR'))
        # Montenegrin is not recognized as an ISO language (yet?) but for now it is
        # unofficially accepted as Serbian from Montenegro
        self.assertEqual(Language.fromopensubtitles('mne'), Language('srp', 'ME'))
        self.assertEqual(Language.fromcode('pob', 'opensubtitles'), Language('por', 'BR'))
        with self.assertRaises(LanguageReverseError):
            Language.fromopensubtitles('zzz')
        with self.assertRaises(LanguageConvertError):
            Language('aaa').opensubtitles
        self.assertEqual(len(get_language_converter('opensubtitles').codes), 606)

        # test with all the languages from the opensubtitles api
        # downloaded from: http://www.opensubtitles.org/addons/export_languages.php
        f = resource_stream('babelfish', 'data/opensubtitles_languages.txt')
        f.readline()
        for l in f:
            idlang, alpha2, _, upload_enabled, web_enabled = l.decode('utf-8').strip().split('\t')
            if not int(upload_enabled) and not int(web_enabled):
                # do not test languages that are too esoteric / not widely available
                continue
            self.assertEqual(Language.fromopensubtitles(idlang).opensubtitles, idlang)
            if alpha2:
                self.assertEqual(Language.fromopensubtitles(idlang), Language.fromopensubtitles(alpha2))
        f.close()

    def test_check_for_better_checks_languages_with_no_current_subs(self):
        superliminal.env.settings.languages.append(Language.fromietf('pt-BR'))
        okensub = {
            'id': 'okensub',
            'language': Language.fromietf('en'),
            'series': "Series Title",
            'season': 2,
            'episode': 3,
            'title': "The Episode",
            'release_group': "OtherRG",
            'content': SUBTITLE_CONTENT
        }
        self.set_subtitles([okensub])
        self.add_video(name="Series.Title.S02E03.720p.WEB-DL.H264-TvRG.mkv")
        yield self.wait_until_processed()

        betterensub = self.transform_sub(okensub, 'betterensub',
            release_group="TvRG", content=SUBTITLE_CONTENT_2)
        okbrsub = self.transform_sub(okensub, 'okbrsub',
            language=Language.fromietf('pt-BR'), content=SUBTITLE_CONTENT_3)
        self.set_subtitles([okensub, betterensub, okbrsub])

        SuperliminalCore.check_for_better()
        yield self.wait_until_processed()
        self.assert_subtitle_contents_matches(expected_content=SUBTITLE_CONTENT_2, suffix='.en.srt')
        self.assert_subtitle_contents_matches(expected_content=SUBTITLE_CONTENT_3, suffix='.pt-BR.srt')

    def test_register_converter(self):
        class TestConverter(LanguageReverseConverter):
            def __init__(self):
                self.to_test = {'fra': 'test1', 'eng': 'test2'}
                self.from_test = {'test1': 'fra', 'test2': 'eng'}

            def convert(self, alpha3, country=None, script=None):
                if alpha3 not in self.to_test:
                    raise LanguageConvertError(alpha3, country, script)
                return self.to_test[alpha3]

            def reverse(self, test):
                if test not in self.from_test:
                    raise LanguageReverseError(test)
                return (self.from_test[test], None)
        language = Language('fra')
        self.assertFalse(hasattr(language, 'test'))
        register_language_converter('test', TestConverter)
        self.assertTrue(hasattr(language, 'test'))
        self.assertIn('test', LANGUAGE_CONVERTERS)
        self.assertEqual(Language('fra').test, 'test1')
        self.assertEqual(Language.fromtest('test2').alpha3, 'eng')
        unregister_language_converter('test')
        self.assertNotIn('test', LANGUAGE_CONVERTERS)
        with self.assertRaises(KeyError):
            Language.fromtest('test1')
        with self.assertRaises(AttributeError):
            Language('fra').test
        clear_language_converters()
        load_language_converters()

    def test_check_for_better_checks_all_recent_videos(self):
        oktvsub = {
            'id': 'oktvsub',
            'language': Language.fromietf('en'),
            'series': "Series Title",
            'season': 2,
            'episode': 3,
            'title': "The Episode",
            'release_group': "OtherRG",
            'content': SUBTITLE_CONTENT
        }
        yield self.add_video(name="Series.Title.S02E03.720p.WEB-DL.H264-TvRG.mkv", subtitle=oktvsub)
        movie_filename = self.create_video_file()
        okmoviesub = {
            'id': 'okmoviesub',
            'language': Language.fromietf('en'),
            'title': "Movie Title",
            'year': 2016,
            'release_group': "OtherRG",
            'content': SUBTITLE_CONTENT
        }
        yield self.add_video(path=movie_filename,
            name="Movie.Title.2016.720p.WEB-DL.H264-MovieRG.mkv", subtitle=okmoviesub)

        bettertvsub = self.transform_sub(oktvsub, 'bettertvsub',
            release_group="TvRG", content=SUBTITLE_CONTENT_2)
        bettermoviesub = self.transform_sub(okmoviesub, 'bettermoviesub',
            release_group="MovieRG", content=SUBTITLE_CONTENT_3)
        self.set_subtitles([oktvsub, bettertvsub, okmoviesub, bettermoviesub])

        SuperliminalCore.check_for_better()
        yield self.wait_until_processed()
        self.assert_subtitle_contents_matches(expected_content=SUBTITLE_CONTENT_2)
        self.assert_subtitle_contents_matches(video_filename=movie_filename,
            expected_content=SUBTITLE_CONTENT_3)

 def test_converter_alpha2(self):
     self.assertEqual(Language('eng').alpha2, 'en')
     self.assertEqual(Language.fromalpha2('en'), Language('eng'))
     self.assertEqual(Language.fromcode('en', 'alpha2'), Language('eng'))
     self.assertRaises(LanguageReverseError, lambda: Language.fromalpha2('zz'))
     self.assertRaises(LanguageConvertError, lambda: Language('aaa').alpha2)
     self.assertEqual(len(language_converters['alpha2'].codes), 184)

 def test_downloads_new_sub_if_new_video_added_for_existing_path(self):
     self.set_subtitles([{
         'id': 'new_sub',
         'language': Language.fromietf('en'),
         'series': "Series Title",
         'season': 2,
         'episode': 3,
         'title': "The Episode",
         'release_group': "TvRG",
         'resolution': '720p',
         'content': SUBTITLE_CONTENT
     },
     {
         'id': 'old_sub',
         'language': Language.fromietf('en'),
         'series': "Series Title",
         'season': 2,
         'episode': 3,
         'title': "The Episode",
         'release_group': "OtherRG",
         'resolution': '720p',
         'content': SUBTITLE_CONTENT_2
     }])
     request = self.get_add_request(name="Series.Title.S02E03.720p.WEB-DL.H264-OtherRG.mkv")
     response = yield self.http_client.fetch(request)
     self.assertEqual(200, response.code)
     yield self.wait_until_processed()
     request = self.get_add_request(name="Series.Title.S02E03.720p.WEB-DL.H264-TvRG.mkv")
     response = yield self.http_client.fetch(request)
     self.assertEqual(200, response.code)
     yield self.wait_until_processed()
     self.assert_subtitle_contents_matches()

def check_missing_subtitle_languages(dirname, filename):
    log.debug("Checking for missing subtitle")
    missing_subtitles = []
    # Check embedded subtitles
    embedded_subtitles = []
    if autosubliminal.SCANEMBEDDEDSUBS:
        embedded_subtitles = _get_embedded_subtitles(dirname, filename)
    # Check default language
    if autosubliminal.DEFAULTLANGUAGE:
        default_language = Language.fromietf(autosubliminal.DEFAULTLANGUAGE)
        # Check with or without alpha2 code suffix depending on configuration
        if autosubliminal.DEFAULTLANGUAGESUFFIX:
            srtfile = os.path.splitext(filename)[0] + u"." + autosubliminal.DEFAULTLANGUAGE + u".srt"
        else:
            srtfile = os.path.splitext(filename)[0] + u".srt"
        if not os.path.exists(os.path.join(dirname, srtfile)) and default_language not in embedded_subtitles:
            missing_subtitles.append(autosubliminal.DEFAULTLANGUAGE)
    # Check additional languages
    if autosubliminal.ADDITIONALLANGUAGES:
        # Always check with alpha2 code suffix for additional languages
        for language in autosubliminal.ADDITIONALLANGUAGES:
            additional_language = Language.fromietf(language)
            srtfile = os.path.splitext(filename)[0] + u"." + language + u".srt"
            if not os.path.exists(os.path.join(dirname, srtfile)) and additional_language not in embedded_subtitles:
                missing_subtitles.append(language)
    return missing_subtitles

def _get_embedded_subtitles(dirname, filename):
    """
    Based on subliminal.video.scan_video(...) but only keep the check for embedded subtitles
    """
    log.debug("Checking for embedded subtitle")
    embedded_subtitle_languages = set()
    try:
        path = os.path.join(dirname, filename)
        if filename.endswith('.mkv'):
            with open(path, 'rb') as f:
                mkv = MKV(f)

            # subtitle tracks
            if mkv.subtitle_tracks:
                for st in mkv.subtitle_tracks:
                    if st.language:
                        try:
                            embedded_subtitle_languages.add(Language.fromalpha3b(st.language))
                        except BabelfishError:
                            log.error('Embedded subtitle track language %r is not a valid language', st.language)
                            embedded_subtitle_languages.add(Language('und'))
                    elif st.name:
                        try:
                            embedded_subtitle_languages.add(Language.fromname(st.name))
                        except BabelfishError:
                            log.debug('Embedded subtitle track name %r is not a valid language', st.name)
                            embedded_subtitle_languages.add(Language('und'))
                    else:
                        embedded_subtitle_languages.add(Language('und'))
                log.debug('Found embedded subtitles %r with enzyme', embedded_subtitle_languages)
            else:
                log.debug('MKV has no subtitle track')
    except:
        log.exception('Parsing video metadata with enzyme failed')
    return embedded_subtitle_languages

 def test_downloads_for_all_configured_languages(self):
     superliminal.env.settings.languages.append(Language.fromietf('pt-BR'))
     self.set_subtitles([{
         'id': 'english_sub',
         'language': Language.fromietf('en'),
         'series': "Series Title",
         'season': 2,
         'episode': 3,
         'title': "The Episode",
         'release_group': "TvRG",
         'resolution': '720p',
         'content': SUBTITLE_CONTENT
     },
     {
         'id': 'brazilian_sub',
         'language': Language.fromietf('pt-BR'),
         'series': "Series Title",
         'season': 2,
         'episode': 3,
         'title': "The Episode",
         'release_group': "TvRG",
         'resolution': '720p',
         'content': SUBTITLE_CONTENT_2
     }])
     request = self.get_add_request(name="Series.Title.S02E03.720p.WEB-DL.H264-TvRG.mkv")
     response = yield self.http_client.fetch(request)
     self.assertEqual(200, response.code)
     yield self.wait_until_processed()
     self.assert_subtitle_contents_matches(expected_content=SUBTITLE_CONTENT, suffix='.en.srt')
     self.assert_subtitle_contents_matches(expected_content=SUBTITLE_CONTENT_2, suffix='.pt-BR.srt')

 def test_converter_alpha3t(self):
     self.assertEqual(Language('fra').alpha3t, 'fra')
     self.assertEqual(Language.fromalpha3t('fra'), Language('fra'))
     self.assertEqual(Language.fromcode('fra', 'alpha3t'), Language('fra'))
     self.assertRaises(LanguageReverseError, lambda: Language.fromalpha3t('zzz'))
     self.assertRaises(LanguageConvertError, lambda: Language('aaa').alpha3t)
     self.assertEqual(len(language_converters['alpha3t'].codes), 418)

    def test_register_converter(self):
        class TestConverter(LanguageReverseConverter):
            def __init__(self):
                self.to_test = {'fra': 'test1', 'eng': 'test2'}
                self.from_test = {'test1': 'fra', 'test2': 'eng'}

            def convert(self, alpha3, country=None, script=None):
                if alpha3 not in self.to_test:
                    raise LanguageConvertError(alpha3, country, script)
                return self.to_test[alpha3]

            def reverse(self, test):
                if test not in self.from_test:
                    raise LanguageReverseError(test)
                return (self.from_test[test], None)
        language = Language('fra')
        self.assertFalse(hasattr(language, 'test'))
        language_converters['test'] = TestConverter()
        self.assertTrue(hasattr(language, 'test'))
        self.assertIn('test', language_converters)
        self.assertEqual(Language('fra').test, 'test1')
        self.assertEqual(Language.fromtest('test2').alpha3, 'eng')
        del language_converters['test']
        self.assertNotIn('test', language_converters)
        self.assertRaises(KeyError, lambda: Language.fromtest('test1'))
        self.assertRaises(AttributeError, lambda: Language('fra').test)

    def test_added_subs_are_returned_by_lang(self):
        path = "/data/Series/Season 1/01 Title.mkv"
        video = Video.fromname("Series.S01E02.Title.720p.WEB-DL.DD5.1.H264-ReleaseGroup.mkv")
        self.datastore.add_video(path, video)

        provider1 = "davessubs"
        sub_id1 = "ABC123"
        lang1 = Language.fromietf("en")
        score1 = 123
        provider2 = "stevesubs"
        sub_id2 = "steve123"
        lang2 = lang1
        score2 = 120
        provider3 = "pablosubs"
        sub_id3 = "umdoistres"
        lang3 = Language.fromietf("pt-BR")
        score3 = 150
        self.datastore.add_download(path, provider1, sub_id1, lang1, score1)
        self.datastore.add_download(path, provider2, sub_id2, lang2, score2)
        self.datastore.add_download(path, provider3, sub_id3, lang3, score3)

        downloads = self.datastore.get_downloads_for_video(path)

        self.assertEqual(
            downloads,
            {
                lang1: [
                    {"provider": provider1, "sub_id": sub_id1, "lang": lang1, "score": score1},
                    {"provider": provider2, "sub_id": sub_id2, "lang": lang2, "score": score2},
                ],
                lang3: [{"provider": provider3, "sub_id": sub_id3, "lang": lang3, "score": score3}],
            },
        )

def getEmbeddedLanguages(video_path):
    embedded_subtitle_languages = set()
    try:
        with open(video_path, 'rb') as f:
            mkv = MKV(f)
            if mkv.subtitle_tracks:
                for st in mkv.subtitle_tracks:
                    if st.language:
                        try:
                            embedded_subtitle_languages.add(Language.fromalpha3b(st.language))
                        except BabelfishError:
                            logger.log('Embedded subtitle track is not a valid language', logger.DEBUG)
                            embedded_subtitle_languages.add(Language('und'))
                    elif st.name:
                        try:
                            embedded_subtitle_languages.add(Language.fromname(st.name))
                        except BabelfishError:
                            logger.log('Embedded subtitle track is not a valid language', logger.DEBUG)
                            embedded_subtitle_languages.add(Language('und'))
                    else:
                        embedded_subtitle_languages.add(Language('und'))
            else:
                logger.log('MKV has no subtitle track', logger.DEBUG)
    except MalformedMKVError:
        logger.log('MKV seems to be malformed, ignoring embedded subtitles', logger.WARNING)

    return embedded_subtitle_languages

def getAlpha3TCode(code):  # We need to make sure that language codes are alpha3T
    """
        :param    code: Alpha2, Alpha3, or Alpha3b code.
        :type     code: C{str}

        :return: Alpha3t language code (ISO 639-2/T) as C{str}
    """
    lang = 'und'
    code = code.strip().lower()

    if len(code) == 3:
        try:
            lang = Language(code).alpha3t
        except:
            try:
                lang = Language.fromalpha3b(code).alpha3t
            except:
                try:
                    lang = Language.fromalpha3t(code).alpha3t
                except:
                    pass

    elif len(code) == 2:
        lang = Language.fromalpha2(code).alpha3t

    return lang

 def test_converter_name(self):
     self.assertEqual(Language('eng').name, 'English')
     self.assertEqual(Language.fromname('English'), Language('eng'))
     self.assertEqual(Language.fromcode('English', 'name'), Language('eng'))
     with self.assertRaises(LanguageReverseError):
         Language.fromname('Zzzzzzzzz')
     self.assertEqual(len(get_language_converter('name').codes), 7874)

def subtitlesLanguages(video_path):
    """Return a list detected subtitles for the given video file"""
    resultList = []
    embedded_subtitle_languages = set()

    # Serch for embedded subtitles
    if not sickbeard.EMBEDDED_SUBTITLES_ALL:
        if video_path.endswith('mkv'):
            try:
                with open(video_path.encode(sickbeard.SYS_ENCODING), 'rb') as f:
                    mkv = MKV(f)
                if mkv.subtitle_tracks:
                    for st in mkv.subtitle_tracks:
                        if st.language:
                            try:
                                embedded_subtitle_languages.add(Language.fromalpha3b(st.language))
                            except BabelfishError:
                                logger.log('Embedded subtitle track is not a valid language', logger.DEBUG)
                                embedded_subtitle_languages.add(Language('und'))
                        elif st.name:
                            try:
                                embedded_subtitle_languages.add(Language.fromname(st.name))
                            except BabelfishError:
                                logger.log('Embedded subtitle track is not a valid language', logger.DEBUG)
                                embedded_subtitle_languages.add(Language('und'))
                        else:
                            embedded_subtitle_languages.add(Language('und'))
                else:
                    logger.log('MKV has no subtitle track', logger.DEBUG)
            except MalformedMKVError:
                logger.log('MKV seems to be malformed, ignoring embedded subtitles', logger.WARNING)

    # Search subtitles in the absolute path
    if sickbeard.SUBTITLES_DIR and ek(os.path.exists, sickbeard.SUBTITLES_DIR):
        video_path = ek(os.path.join, sickbeard.SUBTITLES_DIR, ek(os.path.basename, video_path))
    # Search subtitles in the relative path
    elif sickbeard.SUBTITLES_DIR:
        video_path = ek(os.path.join, ek(os.path.dirname, video_path), sickbeard.SUBTITLES_DIR, ek(os.path.basename, video_path))

    external_subtitle_languages = subliminal.video.scan_subtitle_languages(video_path)
    subtitle_languages = external_subtitle_languages.union(embedded_subtitle_languages)

    if (len(subtitle_languages) is 1 and len(wantedLanguages()) is 1) and Language('und') in subtitle_languages:
        subtitle_languages.remove(Language('und'))
        subtitle_languages.add(fromietf(wantedLanguages()[0]))

    for language in subtitle_languages:
        if hasattr(language, 'opensubtitles') and language.opensubtitles:
            resultList.append(language.opensubtitles)
        elif hasattr(language, 'alpha3') and language.alpha3:
            resultList.append(language.alpha3)
        elif hasattr(language, 'alpha2') and language.alpha2:
            resultList.append(language.alpha2)

    defaultLang = wantedLanguages()

    if ('pob' in defaultLang or 'pb' in defaultLang) and ('pt' not in defaultLang and 'por' not in defaultLang):
            resultList = [x if not x in ['por', 'pt'] else u'pob' for x in resultList]

    return sorted(resultList)

def getEmbeddedLanguages(video_path):
    embedded_subtitle_languages = set()
    try:
        with ek(io.open, video_path, 'rb') as f:
            mkv = MKV(f)
            if mkv.subtitle_tracks:
                for st in mkv.subtitle_tracks:
                    if st.language:
                        try:
                            embedded_subtitle_languages.add(Language.fromalpha3b(st.language))
                        except BabelfishError:
                            logging.debug('Embedded subtitle track is not a valid language')
                            embedded_subtitle_languages.add(Language('und'))
                    elif st.name:
                        try:
                            embedded_subtitle_languages.add(Language.fromname(st.name))
                        except BabelfishError:
                            logging.debug('Embedded subtitle track is not a valid language')
                            embedded_subtitle_languages.add(Language('und'))
                    else:
                        embedded_subtitle_languages.add(Language('und'))
            else:
                logging.debug('MKV has no subtitle track')
    except MalformedMKVError:
        logging.info('MKV seems to be malformed ( %s ), ignoring embedded subtitles' % video_path)

    return embedded_subtitle_languages

 def test_converter_alpha3t(self):
     self.assertEqual(Language('fra').alpha3t, 'fra')
     self.assertEqual(Language.fromalpha3t('fra'), Language('fra'))
     self.assertEqual(Language.fromcode('fra', 'alpha3t'), Language('fra'))
     with self.assertRaises(LanguageReverseError):
         Language.fromalpha3t('zzz')
     with self.assertRaises(LanguageConvertError):
         Language('aaa').alpha3t
     self.assertEqual(len(get_language_converter('alpha3t').codes), 418)

    def query(self, searchword=None):
        params = {"searchword": searchword}
        logger.info("Searching subtitles %r", params)

        r = self.session.get("http://sub.makedie.me/sub/?", params=params, timeout=10)
        r.raise_for_status()

        # loop over
        subtitles = []
        try:
            # BeautifulSoup gives you Unicode, damn it
            soup = BeautifulSoup(r.content, "html5lib")
            results = soup.find_all("div", attrs={"class": "subitem"})
            for it in results:
                releases = it.find("a", attrs={"class": "introtitle"})["title"].strip().split("/")
                sid = re.search("/xml/sub/\d+/(\d+).xml", it.find("a", attrs={"class": "introtitle"})["href"]).group(1)

                download_button = it.find("a", attrs={"id": "downsubbtn"})["onclick"].strip()
                if re.search("location.href\s*=\s*", download_button):
                    link = "http://sub.makedie.me/" + re.search(
                        "location.href\s*=\s*['\"](.*?)['\"]", download_button
                    ).group(1)
                else:
                    logger.debug("No download link found")
                    continue

                lang = ""
                for li in it.find_all("li"):
                    if re.search("格式：", li.text.strip()):
                        subtype = re.search("格式：\s*([^\(]+)(?:\(\?\))*", li.text.strip()).group(1)

                    if re.search("语言：", li.text.strip()):
                        lang = re.search("语言：\s*([^\(]+)(?:\(\?\))*", li.text.strip()).group(1)

                    if re.search("下载次数：", li.text.strip()):
                        downloads = re.search("下载次数：\s*([^\(]+)(?:\(\?\))*", li.text.strip()).group(1)
                        downloads = re.search("(\d+).*", downloads).group(1)

                    if re.search("日期：", li.text.strip()):
                        upload_date = re.search("日期：\s*([^\(]+)(?:\(\?\))*", li.text.strip()).group(1)

                if "简" in lang or "繁" in lang or "双语" in lang:
                    subtitle = MakeDieSubtitle(
                        Language.fromalpha2("zh"), releases, sid, link, subtype, downloads, upload_date
                    )
                else:
                    subtitle = MakeDieSubtitle(
                        Language.fromalpha2("en"), releases, sid, link, subtype, downloads, upload_date
                    )

                logger.debug("Found subtitle %r", subtitle)
                subtitles.append(subtitle)

            return subtitles
        except:
            logger.debug("No subtitle found")
            return []

 def test_converter_alpha2(self):
     self.assertEqual(Language('eng').alpha2, 'en')
     self.assertEqual(Language.fromalpha2('en'), Language('eng'))
     self.assertEqual(Language.fromcode('en', 'alpha2'), Language('eng'))
     with self.assertRaises(LanguageReverseError):
         Language.fromalpha2('zz')
     with self.assertRaises(LanguageConvertError):
         Language('aaa').alpha2
     self.assertEqual(len(get_language_converter('alpha2').codes), 184)