def find_misshing_DH(max_length):

    """
    Run through Ritva Makkot, and search for lines with an unreasonable amount of words until the first period.
    :param max_length:
    :return:
    """
    text={}
    count, lines = 0, 0
    curr_daf=0
    probs = codecs.open('probs_ritva.txt', 'w', 'utf-8')
    files = ["chiddushei one.txt","chiddushei two.txt", "chiddushei three.txt", "chiddushei four.txt", "chiddushei five.txt"]
    for file in files:
        open_file = codecs.open(file, 'r', 'utf-8')
        for line in open_file:
            line = line.replace('\n','')
            if len(line)==0:
                continue
            if line.find(u"#")>=0:
                start=line.find(u"#1")
                end=line.find(u"#2")
                if start>end or start==-1 or end==-1:
                    print '# error'
                daf = line[start:end]
                if daf.find(u'ע"ב')>=0:
                    curr_daf += 1
                elif daf.find(u'דף')>=0:
                    daf = daf.split(u" ")[1]
                    poss_daf = 2*getGematria(daf)-1
                    if poss_daf < curr_daf:
                        print 'daf error'
                    curr_daf = poss_daf
                else:
                    print 'no daf'
            line = line.replace('@1','').replace('@2','')
            words = line.split()

            for index, word in enumerate(words):

                lines += 1

                if word.find(u'.') >= 0:
                    break

                elif index > max_length:
                    probs.write('file: ' + str(file) + "\n")
                    probs.write('current daf:' + AddressTalmud.toStr('en', curr_daf) + "\n")
                    probs.write('line without DH:\t' + ' '.join(words[:max_length]) + "\n\n\n")
                    count += 1
                    break

            else:
                probs.write(u'file: ' + str(file) + u"\n")
                probs.write(u'current daf:' + AddressTalmud.toStr('en', curr_daf) + u"\n")
                probs.write(u'line without DH:\t' + u' '.join(words) + u"\n\n\n")
                count += 1
    print count, lines

def compileCommentaryIntoPage(title, daf):
    page = []
    next = title+" "+AddressTalmud.toStr("en", daf)+".1"
    while next is not None and next.find(AddressTalmud.toStr("en", daf)) >= 0:
        text = get_text_plus(next)
        for line in text['he']:
            page.append(line)

        next = text['next']
    return page

def lookForLineInCommentary(title, daf, line_n):
    total_count = 0
    next = title+" "+AddressTalmud.toStr("en", daf)+":1"
    while next.find(AddressTalmud.toStr("en", daf)) >= 0:
        text = get_text_plus(next)
        local_count = 0
        for line in text['he']:
            local_count+=1
            total_count+=1
            if total_count == line_n:
                return next+"."+str(local_count)
        next = text['next']
    return ""

 def Gemara(self, daf, results):
     self.maharam_line+=1
     self.which_line['gemara']+=1
     if results['gemara'][self.which_line['gemara']] == '0':
         self.missing_ones.append(self.title+" on "+self.masechet+"."+AddressTalmud.toStr("en", daf)+"."+str(self.maharam_line))
     else:
         self.links_to_post.append({
         "refs": [
                  results['gemara'][self.which_line['gemara']],
                 self.title+" on "+self.masechet+"."+AddressTalmud.toStr("en", daf)+"."+str(self.maharam_line)
             ],
         "type": "commentary",
         "auto": True,
         "generated_by": self.title+self.masechet+" linker",
      })

def getLog(siman, result, dh_dict, comm):
    log = []
    for key in result:
        line_n = result[key]
        if line_n[0] == 0:
            append_str = (
                "did not find dh:\n"
                + str(dh_dict[siman][key - 1])
                + "\n in "
                + title_book
                + ", Daf "
                + AddressTalmud.toStr("en", siman)
                + ":"
            )
            append_str += "\nwww.sefaria.org/" + title_book.replace(" ", "_") + "." + AddressTalmud.toStr("en", siman)
            append_str += "\ntext:<b>" + str(dh_dict[siman][key - 1]) + ".</b> " + str(comm[siman][key - 1]) + "\n\n"
            log.append(append_str)
        elif len(line_n) > 1:
            bestGuess = line_n[0]
            guess_str = (
                "looked for dh:\n"
                + str(dh_dict[siman][key - 1])
                + "\n in "
                + title_book
                + ", Daf "
                + AddressTalmud.toStr("en", siman)
            )
            guess_str += " and guessed the dh matches to line " + str(bestGuess) + ":"
            title_c = title_comm.replace(" ", "_")
            guess_str += "\nwww.sefaria.org/" + title_c + "." + AddressTalmud.toStr("en", siman) + "." + str(bestGuess)
            guess_str += "\nbut other options include:\n"
            for guess in line_n:
                if guess != line_n[0]:
                    title = title_book.replace(" ", "_")
                    guess_str += (
                        "line "
                        + str(guess)
                        + ": www.sefaria.org/"
                        + title
                        + "."
                        + AddressTalmud.toStr("en", siman)
                        + "."
                        + str(guess)
                        + " ,\n"
                    )
            guess_str = guess_str[0:-1]
            log.append(guess_str + "\n\n")
    return log

 def Commentary(self, daf, category, results):
     self.maharam_line += 1
     self.which_line[category] += 1
     title = category.title() + " on " + self.masechet
     base_ref = results[category][self.which_line[category]]
     comm_ref = self.title+" on "+self.masechet+"."+AddressTalmud.toStr("en", daf)+"."+str(self.maharam_line)
     if base_ref == '0':
         self.missing_ones.append(comm_ref)
     else:
         self.links_to_post.append({
             "refs": [
                          base_ref,
                         comm_ref
                     ],
             "type": "commentary",
             "auto": True,
             "generated_by": self.title+self.masechet+" linker"
         })
         gemara_ref = self.getGemaraRef(base_ref)
         self.links_to_post.append({
             "refs": [
                 comm_ref,
                 gemara_ref
             ],
             "type": "commentary",
             "auto": True,
             "generated_by": self.title+self.masechet+" linker"
         })

def find_matches(gemara, tosafot, rashi):
    # what needs to be done is to go through each dict and try to match everything, but check each segment that if it is בא"ד
    # ignore if it has a match and match it to previous segment's match
    # and if no match: link with previous segment (as a range) as if this comment really has no DH which is why it has no match

    nones = total = 0
    for pairs in [(tosafot, "Tosafot on Ketubot"), (gemara, "Ketubot"), (rashi, "Rashi on Ketubot")]:
        orig_dict = dict(pairs[0])
        which_dict = pairs[0]
        which_text = pairs[1]
        for daf in which_dict.keys():
            actual_daf = AddressTalmud.toStr("en", daf)
            base_text = TextChunk(Ref("{} {}".format(which_text, actual_daf)), lang='he')
            if not base_text.text:
                continue
            comments = which_dict[daf]
            results = match_ref(base_text, comments, lambda x: x.split(), dh_extract_method=dh_extract)
            for i, result_comment in enumerate(zip(results["matches"], comments)):
                result, comment = result_comment
                comment_wout_bold = comment.replace("<b>", "").replace("</b>", "")
                if u"""בא"ד""" in u" ".join(comment_wout_bold.split()[0:3]) \
                        or u"""שם בא"ד""" in u" ".join(comment_wout_bold.split()[0:3]):
                    results["matches"][i] = results["matches"][i - 1]
            which_dict[daf] = results["matches"]


        for daf in which_dict.keys():
             if which_dict[daf] and orig_dict[daf]:
                which_dict[daf] = create_ranges(orig_dict, which_dict, which_text, daf)
    return gemara, tosafot, rashi

    def getDaf(self, line, current_daf, len_masechet, prev_line):
        prev_num = self.current_daf
        orig_line = line
        line = line.replace("@11 ", "@11")
        if line.split(" ")[0].find('דף')>=0:
            daf_value = getGematria(line.split(" ")[1].replace('"', '').replace("'", ''))
            if line.split(" ")[2].find(self.amud_bet)>=0:
                self.current_daf = 2*daf_value
            else:
                self.current_daf = 2*daf_value - 1
            actual_text = ""
            start_at = 3
            if line.split(" ")[2] not in ['ע"ב', 'ע"א']:
                start_at = 2
            for count, word in enumerate(line.split(" ")):
                if count >= start_at:
                    actual_text += word + " "
        else:
            self.current_daf += 1
            actual_text = line[3:]

        if self.current_daf <= prev_num:
            he_current = AddressTalmud.toStr("he", self.current_daf)
            he_prev = AddressTalmud.toStr("he", prev_num)
            #prev_line = " ".join(prev_line.split(" ")[0:5])
            #orig_line = " ".join(orig_line.split(" ")[0:5])
            print u"{} before {}\n".format(he_prev, he_current)
            self.dont_post = True
            #print u"The line starting: {} is {}\n".format(prev_line, he_prev)
            #print u"It came before the line starting {}, which is {}\n\n".format(orig_line, he_current)


        if not self.current_daf in self.dh1_dict:
            self.dh1_dict[self.current_daf] = []
            for each_cat in self.categories:
                self.dh_by_cat[each_cat][self.current_daf] = []
        self.actual_text = actual_text
        if self.current_daf > len_masechet:
            print "DAF EXTRA {} > {} in {} {}".format(self.current_daf, len_masechet, self.title, self.masechet)
            pass
        self.list_of_dafs.append(self.current_daf)

        return self.current_daf

def post(text, dh_dict, tractate):
     text_array = convertDictToArray(text)
     send_text = {
         "text": text_array,
         "versionTitle": "Ramban on Talmud",
         "versionSource": "http://www.sefaria.org",
         "language": "he"
     }
     post_text("Chiddushei Ramban on "+tractate, send_text)
     links_to_post = []
     daf_array = get_text_plus(tractate)['he']
     match = Match(in_order=True, min_ratio=80, guess=False, range=True, can_expand=False)
     for daf in sorted(dh_dict.keys()):
         dh_list = dh_dict[daf]
         results = match.match_list(dh_list, daf_array[daf-1], tractate+" "+AddressTalmud.toStr("en", daf))
         for key, value in results.iteritems():
             value = value.replace("0:", "")
             talmud_end = tractate + "." + AddressTalmud.toStr("en", daf) + "." + value
             ramban_end = "Chiddushei_Ramban_on_" + tractate + "." + AddressTalmud.toStr("en", daf) + "." + str(key)
             links_to_post.append({'refs': [talmud_end, ramban_end], 'type': 'commentary', 'auto': 'True', 'generated_by': "ramban"+tractate})
     post_link(links_to_post)

def post(text, dh_dict, tractate):
     text_array = convertDictToArray(text)
     send_text = {
         "text": text_array,
         "versionTitle": "Chiddushei HaRamban, Jerusalem 1928-29",
         "versionSource": "http://primo.nli.org.il/primo_library/libweb/action/dlDisplay.do?vid=NLI&docId=NNL_ALEPH001294828",
         "language": "he"
     }
     post_text("Chiddushei Ramban on "+tractate, send_text)
     links_to_post = []

     for daf in sorted(dh_dict.keys()):
         dh_list = dh_dict[daf]
         daf_text = Ref(tractate+" "+AddressTalmud.toStr("en", daf)).text('he').text
         results = match.match_list(dh_list, daf_text, tractate+" "+AddressTalmud.toStr("en", daf))
         for key, value in results.iteritems():
             value = value.replace("0:", "")
             talmud_end = tractate + "." + AddressTalmud.toStr("en", daf) + "." + value
             talmud_end = tractate + "." + AddressTalmud.toStr("en", daf) + "." + value
             ramban_end = "Chiddushei_Ramban_on_" + tractate + "." + AddressTalmud.toStr("en", daf) + "." + str(key)
             links_to_post.append({'refs': [talmud_end, ramban_end], 'type': 'commentary', 'auto': 'True', 'generated_by': "ramban"+tractate})
     post_link(links_to_post)

 def Rosh(self, perek, daf, dh, results):
     self.maharam_line += 1
     self.rosh_line += 1
     if results[perek-1][self.rosh_line]:
         self.links_to_post.append({
             "refs": [
                      results[perek-1][self.rosh_line].normal(),
                     self.title+" on "+self.masechet+"."+AddressTalmud.toStr("en", daf)+"."+str(self.maharam_line)
                 ],
             "type": "commentary",
             "auto": True,
             "generated_by": self.title+self.masechet+" linker",
          })

 def postLinks(self):
     def base_tokenizer(str):
         str = re.sub(ur"\([^\(\)]+\)", u"", str)
         word_list = re.split(ur"\s+", str)
         word_list = [w for w in word_list if w]  # remove empty strings
         return word_list
     def dh_extract_method(str):
         str = str.replace(u'בד"ה', u'').replace(u'וכו', u'')
         return str
     '''
     1. strip out "" from dhs with list comprehension
     2. make dictionary where each dh str is key and the value is its index in the array
     '''
     links = []
     for daf in self.text:
         dhs_arr = [dh for dh in self.dhs[daf] if len(dh) > 0]
         gemara_text = Ref("{} {}".format(self.tractate, AddressTalmud.toStr("en", daf))).text('he')
         results = match_ref(gemara_text, dhs_arr, base_tokenizer, dh_extract_method=dh_extract_method, verbose=False)['matches']
         self.makeDicts(daf)
         rashba_refs = []
         for dh in dhs_arr:
             rashba_refs.append("Rashba on {} {}.{}".format(self.tractate, AddressTalmud.toStr("en", daf), self.dh_dict[daf][dh]+1))
         link_pairs = zip(rashba_refs, results)
         for link_pair in link_pairs:
             if link_pair[1]:
                 links.append(
                 {
                 "refs": [
                              link_pair[0],
                             link_pair[1].normal()
                         ],
                 "type": "commentary",
                 "auto": True,
                 "generated_by": "rashba{}".format(self.tractate)
                 }
                 )
     post_link(links, server=self.server)

def match_and_link(text, masechet):
	match = Match(in_order=True, min_ratio=80, guess=False, range=True, can_expand=False)
	for daf_count, daf in enumerate(text):
		dhs = []
		comments = []
		for each_line in daf:
			if each_line.find("כו'") >= 0:
				dh, comment = each_line.split("כו'", 1)
			elif each_line.find(".") >= 0:
				dh, comment = each_line.split(".", 1)
			else:
				dh, comment = splitText(each_line, 10)
			dhs.append(dh)
			comments.append(comment)
		pdb.set_trace()
		talmud_text = get_text_plus(masechet+"."+AddressTalmud.toStr("en", daf_count+3))['he']
		result = match.match_list(dhs, talmud_text)

 def getTC(self, category, daf, masechet):
     if category in ["tosafot", "ran", "rosh"]:
         title = "{} on {}".format(category.title(), masechet)
         return Ref(title+"."+AddressTalmud.toStr("en", daf)).text('he')
     elif category == "gemara":
         return Ref(masechet+" "+AddressTalmud.toStr("en", daf)).text('he')
     elif category == "rashi":
         rashi = Ref("Rashi on "+self.masechet+"."+AddressTalmud.toStr("en", daf)).text('he')
         if len(rashi.text) == 0:
             print "rashbam by default {} {}".format(masechet, AddressTalmud.toStr("en", daf))
             return Ref("Rashbam on "+self.masechet+"."+AddressTalmud.toStr("en", daf)).text('he')
         else:
             return rashi
     elif category == "rashbam":
         print "rashbam {} {}".format(masechet, AddressTalmud.toStr("en", daf))
         return Ref("Rashbam on "+self.masechet+"."+AddressTalmud.toStr("en", daf)).text('he')

		print 'line did not start with 11'

match_obj=Match(in_order=False, min_ratio=80, guess=False, range=False)

last_daf = max(comm_dict.keys())
param = "off"
for daf in comm_dict:
	if daf==last_daf:
		param = "on"
	send_text = {
				"versionTitle": "Maharam Shif on "+masechet,
				"versionSource": "http://www.sefaria.org",
				"language": "he",
				"text": comm_dict[daf],
				}
	post_text("Maharam Shif on "+masechet+"."+AddressTalmud.toStr("en", daf), send_text, param)


for category in categories:
  if category=='paragraph':
  	continue
  elif category=='gemara':
  	title = masechet
  elif category=='rashi':
  	title = "Rashi on "+masechet
  elif category=='tosafot':
  	title = "Tosafot on "+masechet
  	
  for daf in dh_dict[category]:
  	dh_arr = dh_dict[category][daf]
  	text = compileCommentaryIntoPage(title, daf)

guess = 0
matched = 0
log = []
dh_dict = {}
rashi_comments = {}
prev_line = 0

title_book = "Keritot"
title_comm = "Rashi on Keritot"

for i in range(54):
    count = 0
    rashi_comments[i + 3] = []
    dh_dict[i + 3] = []
    he_daf = u"כריתות_"
    he_daf += AddressTalmud.toStr("he", i + 3)
    he_daf = he_daf.replace(u"\u05f4", u"")
    he_daf = he_daf.replace(u"׳", u"")
    he_daf = he_daf.replace(" ", "_")
    he_daf = he_daf + ".txt"
    f = open("../Noah-Santacruz-rashiPosting/Rashi/" + he_daf, "r")
    for line in f:
        line = line.replace("\n", "")
        something = line.replace(" ", "")
        if len(something) > 0:
            if count % 2 == 0:
                dh_dict[i + 3].append(line)
            else:
                rashi_comments[i + 3].append(line)
            count += 1
    f.close()

matched = 0
log = []
dh_dict = {}
rashi_comments = {}
prev_line = 0

title_book = "Avodah Zarah"
title_comm = "Tosafot on Avodah Zarah" 

for j in range(2): 
	i = j+
	count = 0
	rashi_comments[i+3] = []
	dh_dict[i+3] = []
	he_daf = u"עבודה_זרה_"
	he_daf += AddressTalmud.toStr("he", i+3)
	he_daf = he_daf.replace(u"\u05f4", u"")
	he_daf = he_daf.replace(u"׳", u"")
	he_daf = he_daf.replace(" ", "_")
	he_daf = he_daf + ".txt"
	f = open("../Noah-Santacruz-rashiPosting/Tosafot/"+he_daf, 'r')
	for line in f:
		line = line.replace("\n", "")
		something = line.replace(" ", "")
		if len(something) > 0:
			if count % 2 == 0:
				dh_dict[i+3].append(line)
			else:
				rashi_comments[i+3].append(line)
			count+=1
	f.close()		

comm = {}
book = {}
total = 0
non_match = 0
guess = 0
matched = 0
log = []
dh_dict = {}
rashi_comments = {}
prev_line = 0
for i in range(150): #152 
	count = 0
	rashi_comments[i+3] = []
	dh_dict[i+3] = []
	he_daf = u"עבודה זרה_"
	he_daf += AddressTalmud.toStr("he", i+3)
	he_daf = he_daf.replace(u"\u05f4", u"")
	he_daf = he_daf.replace(u"׳", u"")
	he_daf = he_daf.replace(" ", "_")
	he_daf = he_daf + ".txt"
	f = open("../Noah-Santacruz-rashiPosting/Tosafot/"+he_daf, 'r')
	for line in f:
		line = line.replace("\n", "")
		something = line.replace(" ", "")
		if len(something) > 0:
			if count % 2 == 0:
				dh_dict[i+3].append(line)
			else:
				if line.find(" - ")==-1:
					line = line.replace(".", " - ", 1)
				rashi_comments[i+3].append(line)

			"versionSource": "http://www.sefaria.org/",
			"language": "he",
			"text": [comm],
			}
		post_text("Yad Ramah on Bava Batra, Perek "+str(current_perek)+", Comment "+str(comment_key), text)

match_obj=Match(in_order=True, min_ratio=80, guess=False, range=True)
skipped_arr = []
result = {}
for current_perek in range(10):
	current_perek+=1
	print current_perek
	search_for = 0
	for daf in sorted(daf_dict[current_perek].keys()):			
		print daf
		text = get_text("Bava Batra."+AddressTalmud.toStr("en", daf))
		dh_list = daf_dict[current_perek][daf]
		result[daf] = match_obj.match_list(dh_list, text, "Bava Batra "+AddressTalmud.toStr("en", daf))
		print result[daf]
		for key in result[daf]:
			if result[daf][key].find("0:") >= 0:
				result[daf][key] = result[daf][key].replace("0:","")
			search_for += 1
			line_n = result[daf][key]
			count = 0
			for comment_key in comments_order[current_perek]:
				count+=1
				if comment_key not in comm_dict[current_perek]:
					if comment_key not in skipped_arr:
						search_for+=1
						skipped_arr.append(comment_key)

			comm_dict[daf].append(comm)
			if hasTags(comm) or hasTags(dh) or hasTags(before_dh):
				pdb.set_trace()
			if just_added_dh == False:
				dh_dict[daf].append("")
			just_added_dh = False
		before_dh =""
		temp_text = ""
result = {}
guess=0
no_guess=0
for daf in dh_dict.keys():
	if len(dh_dict[daf]) != len(comm_dict[daf]):
		pdb.set_trace()
for daf in dh_dict.keys():
	text = get_text("Gittin."+AddressTalmud.toStr("en", daf))
	try:
		match_obj=Match(in_order=True, min_ratio=70, guess=False, range=True, maxLine=len(text)-1)
	except:
		pdb.set_trace()
	dh_arr = []
	for i in range(len(dh_dict[daf])):
		if len(dh_dict[daf][i]) > 0:
			dh_arr.append(dh_dict[daf][i])
	result[daf] = match_obj.match_list(dh_arr, text)
	dh_count = 1
	'''
	if len(dh_dict[daf][i]) == 0, then comm_dict[daf][i] gets added to comm_dict[daf][i-1]+"<br>"
	'''
	for i in range(len(comm_dict[daf])):
		 if (daf, i) in before_dh_dict: