def test_simple_replace(self):
        set_1 = SubtitleSet.from_list('en', [
         (0, 1000, "Hey 1"),
         (1000, 2000, "Hey 2"),
         (2000, 3000, "Hey 3"),
         (3000, 4000, "Hey 4"),
        ])
        set_2 = SubtitleSet.from_list('en', [
         (0, 1000, "Hey 1"),
         (1000, 2000, "Hey New 2"),
         (2000, 3000, "Hey 3"),
         (3000, 4000, "Hey 4"),
        ])
        result = diff(set_1, set_2)
        self.assertEqual(result['changed'], True)
        self.assertAlmostEqual(result['time_changed'], 0)
        # for text_changed, we calculate as follows: there are 8 total subs.
        # 6 of those are matches and 1 is different in both sets.  So 2/8.0
        # has been changed.
        self.assertAlmostEqual(result['text_changed'], 2/8.0)
        self.assertEqual(len(result['subtitle_data']), 4)

        # check the lines that haven't changed
        self.check_unchanged_subtitle_data(result, set_1, set_2, 0, 2, 3)
        # check the line that was inserted
        insert_sub_data = result['subtitle_data'][1]
        self.assertEquals(insert_sub_data['time_changed'], False)
        self.assertEquals(insert_sub_data['text_changed'], True)
        self.assertEquals(insert_sub_data['subtitles'][0], set_1[1])
        self.assertEquals(insert_sub_data['subtitles'][1], set_2[1])

    def merge_subtitles(cls, subtitle_sets, initial_ttml=None):
        """Combine multiple subtitles sets into a single XML string.
        """
        if len(subtitle_sets) == 0:
            raise TypeError("DFXPGenerator.merge_subtitles: No subtitles given")

        if initial_ttml is None:
            tt = SubtitleSet('').as_etree_node()
            body = tt.find(TTML + 'body')
            body.remove(body.find(TTML + 'div'))
        else:
            tt = initial_ttml
            body = tt.find(TTML + 'body')
            if body is None:
                raise ValueError("no body tag")

        # set the default language to blank.  We will create a div for each
        # subtitle set and set xml:lang on that.
        tt.set(XML + 'lang', '')

        # for each subtitle set we will append the body of tt
        for i, subtitle_set in enumerate(subtitle_sets):
            root_elt = subtitle_set.as_etree_node()
            language_code = root_elt.get(XML + 'lang')
            lang_div = etree.SubElement(body, TTML + 'div')
            lang_div.set(XML + 'lang', language_code)
            lang_div.extend(root_elt.find(TTML + 'body').findall(TTML + 'div'))
        utils.indent_ttml(tt)
        return etree.tostring(tt)

    def test_replace_multiple_lines_with_single(self):
        set_1 = SubtitleSet.from_list('en', [
         (0, 1000, "Hey 1"),
         (1000, 2000, "Hey 2"),
         (2000, 3000, "Hey 3"),
         (3000, 4000, "Hey 4"),
        ])
        set_2 = SubtitleSet.from_list('en', [
         (0, 1000, "Hey 1"),
         (1000, 3000, "Hey 2 and 3"),
         (3000, 4000, "Hey 4"),
        ])
        result = diff(set_1, set_2)
        self.assertEqual(result['changed'], True)
        # for both time_change and text_changed, we calculate them as follows:
        # there are 7 total subs.  4 of those are matches and 2 in set_1 were
        # replaced with 1 in set_2.  So the change amount is 3/7.
        self.assertAlmostEqual(result['time_changed'], 3/7.0)
        self.assertAlmostEqual(result['text_changed'], 3/7.0)
        self.assertEqual(len(result['subtitle_data']), 4)

        # check the lines that haven't changed
        self.check_unchanged_subtitle_data(result, set_1, set_2, 0, 3)
        # check the line that was inserted
        line1 = result['subtitle_data'][1]
        self.assertEquals(line1['time_changed'], True)
        self.assertEquals(line1['text_changed'], True)
        self.assertEquals(line1['subtitles'][0], set_1[1])
        self.assertEquals(line1['subtitles'][1], set_2[1])
        line2 = result['subtitle_data'][2]
        self.assertEquals(line2['time_changed'], True)
        self.assertEquals(line2['text_changed'], True)
        self.assertEquals(line2['subtitles'][0], set_1[2])
        self.assertEquals(line2['subtitles'][1], self.empty_line())

    def test_insert(self):
        set_1 = SubtitleSet.from_list('en', [
         (0, 1000, "Hey 1"),
         (1000, 2000, "Hey 2"),
         (2000, 3000, "Hey 3"),
         (3000, 4000, "Hey 4"),
        ])
        set_2 = SubtitleSet.from_list('en', [
         (0, 1000, "Hey 1"),
         (500, 800, "Hey 1.5"),
         (1000, 2000, "Hey 2"),
         (2000, 3000, "Hey 3"),
         (3000, 4000, "Hey 4"),
        ])
        result = diff(set_1, set_2)
        self.assertEqual(result['changed'], True)
        # for both time_change and text_changed, we calculate them as follows:
        # there are 9 total subs.  8 of those are matches and 1 is new in
        # set_2.  So the change amount is 1/9
        self.assertAlmostEqual(result['time_changed'], 1/9.0)
        self.assertAlmostEqual(result['text_changed'], 1/9.0)
        self.assertEqual(len(result['subtitle_data']), 5)

        # check the lines that haven't changed
        self.check_unchanged_subtitle_data(result, set_1, set_2, 0, 2, 3, 4)
        # check the line that was inserted
        insert_sub_data = result['subtitle_data'][1]
        self.assertEquals(insert_sub_data['time_changed'], True)
        self.assertEquals(insert_sub_data['text_changed'], True)
        self.assertEquals(insert_sub_data['subtitles'][0], self.empty_line())
        self.assertEquals(insert_sub_data['subtitles'][1], set_2[1])

class JSONParser(BaseTextParser):
    file_type = 'json'

    def __init__(self, input_string, pattern, language=None, flags=[], eager_parse=True):
        self.input_string = input_string
        self.pattern = pattern
        self.language = language
        super(JSONParser, self).__init__(input_string, pattern, language=language,
            flags=[], eager_parse=eager_parse)

    def to_internal(self):
        if not hasattr(self, 'sub_set'):
            self.sub_set = SubtitleSet(self.language)

            try:
                data = json.loads(self.input_string)
            except ValueError:
                raise SubtitleParserError("Invalid JSON data provided.")

            # Sort by the ``position`` key
            data = sorted(data, key=lambda k: k['position'])

            for sub in data:
                self.sub_set.append_subtitle(sub['start'], sub['end'],
                    sub['text'])

        return self.sub_set

 def setUp(self):
     self.en_subs = SubtitleSet('en')
     self.es_subs = SubtitleSet('es')
     self.fr_subs = SubtitleSet('fr')
     self.en_subs.append_subtitle(1000, 1500, 'content')
     self.es_subs.append_subtitle(1000, 1500, 'spanish content')
     self.es_subs.append_subtitle(2000, 2500, 'spanish content 2',
                                  new_paragraph=True)
     self.fr_subs.append_subtitle(1000, 1500, 'french content')

 def test_calc_changes(self):
     set_1 = SubtitleSet.from_list(
         "en", [(0, 1000, "Hey 1"), (1000, 2000, "Hey 2"), (2000, 3000, "Hey 3"), (3000, 4000, "Hey 4")]
     )
     set_2 = SubtitleSet.from_list(
         "en", [(0, 1000, "Hey 1"), (1000, 2000, "Hey New 2"), (2000, 3000, "Hey 3"), (3000, 4000, "Hey 4")]
     )
     text_changed, time_changed = calc_changes(set_1, set_2)
     self.assertAlmostEqual(time_changed, 0)
     self.assertAlmostEqual(text_changed, 2 / 8.0)

    def test_data_ordering(self):
        set_1 = SubtitleSet.from_list("en", [(0, 1000, "Hey 1")])
        set_2 = SubtitleSet.from_list("en", [(0, 1000, "Hey 1"), (1200, 2000, "Hey 2"), (2000, 3000, "Hey 3")])
        result = diff(set_1, set_2)

        subs_result = result["subtitle_data"][2]["subtitles"]
        # make sure the 0 index subs is for set_1, test
        # we respect the ordering of arguments passed to diff
        self.assertEqual(subs_result[0].text, None)
        self.assertEqual(subs_result[1].text, "Hey 3")

    def test_unsynced_reflect_time_changes(self):
        set_1 = SubtitleSet.from_list('en', [
            (0, 1000, "Hey 1"),
            ])
        set_2 = SubtitleSet.from_list('en', [
            (0, 1000, "Hey 1"),
            (None, None, "Hey 2"),
            ])
        result = diff(set_1, set_2)

        self.assertAlmostEqual(result['time_changed'], 1/3.0)

class YoutubeParser(BaseTextParser):

    file_type = 'youtube'

    def __init__(self, input_string, language_code):
        self.language_code = language_code
        self._pattern = None

        self.input_string = input_string
        self.language = language_code

    def __iter__(self):
        if not hasattr(self, 'sub_set'):
            self.to_internal()

        for sub in self.sub_set:
            yield sub

    def to_internal(self):
        if not hasattr(self, 'sub_set'):
            try:
                self.sub_set = SubtitleSet(self.language)
                xml = etree.fromstring(self.input_string.encode('utf-8'))

                has_subs = False
                total_items = len(xml)
                for i,item in enumerate(xml):
                    duration = 0
                    start = int(float(item.get('start')) * 1000)
                    if hasattr(item, 'duration'):
                        duration = int(float(item.get('dur', 0)) * 1000)
                    elif i+1 < total_items:
                        # youtube sometimes omits the duration attribute
                        # in this case we're displaying until the next sub
                        # starts
                        next_item = xml[i+1]
                        duration = int(float(next_item.get('start')) * 1000) - start
                    else:
                        # hardcode the last sub duration at 3 seconds
                        duration = 3000
                    end = start + duration
                    text = item.text and unescape_html(item.text) or u''
                    self.sub_set.append_subtitle(start, end, text)
                    has_subs = True
                if not has_subs:
                    raise ValueError("No subs")
            except Exception as e:
                raise SubtitleParserError(original_error=e)


        return self.sub_set

 def test_text_changes(self):
     set_1 = SubtitleSet.from_list(
         "en", [(0, 1000, "Hey 1"), (1000, 2000, "Hey 2"), (2000, 3000, "Hey 3"), (3000, 4000, "Hey 4")]
     )
     set_2 = SubtitleSet.from_list(
         "en", [(0, 1000, "Hey 1"), (1000, 2000, "Hey 22"), (2000, 3000, "Hey 3"), (3000, 4000, "Hey 4")]
     )
     result = diff(set_1, set_2)
     self.assertEqual(result["changed"], True)
     self.assertEqual(result["text_changed"], 1 / 4.0)
     self.assertEqual(result["time_changed"], 0)
     self.assertEqual(len(result["subtitle_data"]), 4)
     # only sub #2 should have text changed
     for i, sub_data in enumerate(result["subtitle_data"]):
         self.assertEqual(sub_data["text_changed"], i == 1)

 def test_one_set_empty(self):
     set_1 = SubtitleSet.from_list(
         "en", [(0, 1000, "Hey 1"), (1000, 2000, "Hey 2"), (2000, 3000, "Hey 3"), (3000, 4000, "Hey 4")]
     )
     result = diff(set_1, SubtitleSet("en"))
     self.assertEqual(result["changed"], True)
     self.assertEqual(result["text_changed"], 1.0)
     self.assertEqual(result["time_changed"], 1.0)

    def test_unsynced_generator(self):
        subs = SubtitleSet('en')
        for x in xrange(0,5):
            subs.append_subtitle(None, None,"%s" % x)
        output = unicode(SBVGenerator(subs))

        parsed = SBVParser(output,'en')
        internal = parsed.to_internal()

        subs = [x for x in internal.subtitle_items()]
        self.assertEqual(len(internal), 5)
        for i,sub in enumerate(subs):
            self.assertEqual(sub[0], None )
            self.assertEqual(sub[1], None )
        generated = SBVGenerator(internal)
        self.assertEqual(generated.format_time(None), u'9:59:59.990')
        self.assertIn(u'''9:59:59.990,9:59:59.990\r\n0\r\n\r\n9:59:59.990,9:59:59.990\r\n1\r\n\r\n9:59:59.990,9:59:59.990\r\n2\r\n\r\n9:59:59.990,9:59:59.990\r\n3\r\n\r\n9:59:59.990,9:59:59.990\r\n4\r\n''',
            unicode(generated))

    def test_unsynced_generator(self):
        subs = SubtitleSet('en')
        for x in xrange(0,5):
            subs.append_subtitle(None, None,"%s" % x)
        output = unicode(DFXPGenerator(subs))

        parsed = DFXPParser(output, 'en')
        internal = parsed.to_internal()

        subs = [x for x in internal.subtitle_items()]
        self.assertEqual(len(internal), 5)
        for i,sub in enumerate(subs):
            self.assertIsNone(sub[0])
            self.assertIsNone(sub[1])
            self.assertEqual(sub[2], str(i))

        for node in internal.get_subtitles():
            self.assertIsNone(get_attr(node, 'begin'))
            self.assertIsNone(get_attr(node, 'end'))

    def test_unsynced_generator(self):
        subs = SubtitleSet("en")
        for x in xrange(0, 5):
            subs.append_subtitle(None, None, "%s" % x)
        output = unicode(SBVGenerator(subs, language="en"))

        parsed = SBVParser(output, "en")
        internal = parsed.to_internal()

        subs = [x for x in internal.subtitle_items()]
        self.assertEqual(len(internal), 5)
        for i, sub in enumerate(subs):
            self.assertEqual(sub[0], None)
            self.assertEqual(sub[1], None)
        generated = SBVGenerator(internal)
        self.assertEqual(generated.format_time(None), u"9:59:59.000")
        self.assertIn(
            u"""9:59:59.000,9:59:59.000\r\n0\r\n\r\n9:59:59.000,9:59:59.000\r\n1\r\n\r\n9:59:59.000,9:59:59.000\r\n2\r\n\r\n9:59:59.000,9:59:59.000\r\n3\r\n\r\n9:59:59.000,9:59:59.000\r\n4\r\n""",
            unicode(generated),
        )

class TXTParser(BaseTextParser):

    file_type = 'txt'

    _linebreak_re = re.compile(r"\n\n|\r\n\r\n|\r\r")

    def __init__(self, input_string, language=None, linebreak_re=_linebreak_re, eager_parse=True):
        self.language = language
        self.input_string = linebreak_re.split(input_string)

        if eager_parse:
            self.to_internal()

    def __len__(self):
        return len(self.input_string)

    def __nonzero__(self):
        return bool(self.input_string)

    def _result_iter(self):
        for item in self.input_string:
            output = {}
            output['start'] = None
            output['end'] = None
            output['text'] = utils.strip_tags(item)
            yield output

    def to_internal(self):

        if not hasattr(self, 'sub_set'):
            self.sub_set = SubtitleSet(self.language)
            valid = False
            for item in self._result_iter():
                item['text'] = item['text'].replace("\n", '<br/>')
                if not valid and ''.join(item['text'].split()):
                    valid = True
                self.sub_set.append_subtitle(item['start'], item['end'],
                                             item['text'], escape=False)
            if not valid:
                raise SubtitleParserError("No subs")
        return self.sub_set

    def to_internal(self):

        if not hasattr(self, 'sub_set'):
            self.sub_set = SubtitleSet(self.language)
            valid = False
            for item in self._result_iter():
                if not valid and ''.join(item['text'].split()):
                    valid = True
                self.sub_set.append_subtitle(item['start'], item['end'], item['text'])
            if not valid:
                raise SubtitleParserError("No subs")
        return self.sub_set

    def test_delete(self):
        set_1 = SubtitleSet.from_list(
            "en", [(0, 1000, "Hey 1"), (1000, 2000, "Hey 2"), (2000, 3000, "Hey 3"), (3000, 4000, "Hey 4")]
        )
        set_2 = SubtitleSet.from_list("en", [(0, 1000, "Hey 1"), (2000, 3000, "Hey 3"), (3000, 4000, "Hey 4")])
        result = diff(set_1, set_2)
        self.assertEqual(result["changed"], True)
        # for both time_change and text_changed, we calculate them as follows:
        # there are 7 total subs.  6 of those are matches and 1 is new in
        # set_2.  So the change amount is 1/9
        self.assertAlmostEqual(result["time_changed"], 1 / 7.0)
        self.assertAlmostEqual(result["text_changed"], 1 / 7.0)
        self.assertEqual(len(result["subtitle_data"]), 4)

        # check the lines that haven't changed
        self.check_unchanged_subtitle_data(result, set_1, set_2, 0, 2, 3)
        # check the line that was deleted
        delete_sub_data = result["subtitle_data"][1]
        self.assertEquals(delete_sub_data["time_changed"], True)
        self.assertEquals(delete_sub_data["text_changed"], True)
        self.assertEquals(delete_sub_data["subtitles"][1], self.empty_line())
        self.assertEquals(delete_sub_data["subtitles"][0], set_1[1])

 def test_time_changes(self):
     set_1 = SubtitleSet.from_list('en', [
      (0, 1000, "Hey 1"),
      (1000, 2000, "Hey 2"),
      (2000, 3000, "Hey 3"),
      (3000, 4000, "Hey 4"),
     ])
     set_2 = SubtitleSet.from_list('en', [
      (0, 1000, "Hey 1"),
      (1200, 2000, "Hey 2"),
      (2000, 3000, "Hey 3"),
      (3000, 4000, "Hey 4"),
     ])
     result = diff(set_1, set_2)
     self.assertEqual(result['changed'], True)
     self.assertEqual(result['time_changed'], 1/4.0)
     self.assertEqual(result['text_changed'], 0)
     self.assertEqual(len(result['subtitle_data']), 4)
     # only sub #2 should have text changed
     for i,sub_data in enumerate(result['subtitle_data']):
         self.assertEqual(sub_data['time_changed'], i ==1)
         self.assertFalse(sub_data['text_changed'])

    def to_internal(self):
        if not hasattr(self, 'sub_set'):
            match = None
            try:
                self.sub_set = SubtitleSet(self.language)
                for match in self._matches:
                    item = self._get_data(match.groupdict())
                    text = self.get_markup(item['text'])
                    self.sub_set.append_subtitle(item['start'], item['end'], text, escape=False)
                if match is None:
                    raise ValueError("No subs found")
            except Exception as e:
                raise SubtitleParserError(original_error=e)

        return self.sub_set