def test_utf8_excerpt(self):
     """Characters should stay in UTF-8."""
     wc = WikiClient()
     page = Document.objects.get(pk=4)
     q = u'fa\xe7on'
     excerpt = wc.excerpt(page.html, q)
     assert q in excerpt, u'%s not in %s' % (q, excerpt)

    def test_no_syntax_error(self):
        """Test that special chars cannot cause a syntax error."""
        wc = WikiClient()
        results = wc.query('video^$')
        eq_(1, len(results))

        results = wc.query('video^^^$$$^')
        eq_(1, len(results))

    def test_exclude_words(self):
        """Excluding words with -word works."""
        wc = WikiClient()
        results = wc.query('spanish')
        eq_(1, len(results))

        results = wc.query('spanish -content')
        eq_(0, len(results))

 def test_unicode_excerpt(self):
     """Unicode characters in the excerpt should not be a problem."""
     wc = WikiClient()
     page = Document.objects.get(pk=2)
     try:
         excerpt = wc.excerpt(page.html, u'\u3068')
         render('{{ c }}', {'c': excerpt})
     except UnicodeDecodeError:
         self.fail('Raised UnicodeDecodeError.')

 def test_range_filter(self):
     """Test filtering on a range."""
     wc = WikiClient()
     filter_ = ({'filter': 'updated',
                 'max': 1244355125,
                 'min': 1244355115,
                 'range': True},)
     results = wc.query('', filter_)
     eq_(1, len(results))

 def test_clean_excerpt(self):
     """SearchClient.excerpt() should not allow disallowed HTML through."""
     wc = WikiClient()  # Index strips HTML
     qc = QuestionsClient()  # Index does not strip HTML
     input = "test <div>the start of something</div>"
     output_strip = "<b>test</b>  the start of something"
     output_nostrip = "<b>test</b> &lt;div&gt;the start of " "something&lt;/div&gt;"
     eq_(output_strip, wc.excerpt(input, "test"))
     eq_(output_nostrip, qc.excerpt(input, "test"))

 def test_range_filter(self):
     """Test filtering on a range."""
     wc = WikiClient()
     filter_ = ({'filter': 'updated',
                 'max': 1285765791,
                 'min': 1284664176,
                 'range': True},)
     results = wc.query('', filter_)
     eq_(2, len(results))

    def test_translations_inherit_os_values(self):
        wc = WikiClient()
        filters = [{"filter": "locale", "value": (crc32("fr"),)}, {"filter": "os", "value": (1,)}]
        results = wc.query("", filters)
        eq_(1, len(results))
        eq_(4, results[0]["id"])

        filters[1]["value"] = (4,)
        results = wc.query("", filters)
        eq_(0, len(results))

    def test_translations_inherit_os_values(self):
        wc = WikiClient()
        filters = [{'filter': 'locale', 'value': (crc32('fr'),)},
                   {'filter': 'os', 'value': (1,)}]
        results = wc.query('', filters)
        eq_(1, len(results))
        eq_(4, results[0]['id'])

        filters[1]['value'] = (4,)
        results = wc.query('', filters)
        eq_(0, len(results))

 def test_unicode_excerpt(self):
     """Unicode characters in the excerpt should not be a problem."""
     wc = WikiClient()
     q = 'contribute'
     results = wc.query(q)
     eq_(1, len(results))
     page = WikiPage.objects.get(pk=results[0]['id'])
     try:
         excerpt = wc.excerpt(page.content, q)
         render('{{ c }}', {'c': excerpt})
     except UnicodeDecodeError:
         self.fail('Raised UnicodeDecodeError.')

 def test_clean_hyphens(self):
     """Hyphens in words aren't special characters."""
     wc = WikiClient()
     results = wc.query('marque-page')
     eq_(1, len(results))

 def test_ngram_chars(self):
     """Ideographs are handled correctly."""
     wc = WikiClient()
     results = wc.query(u'\u30c1')
     eq_(1, len(results))
     eq_(2, results[0]['id'])

 def test_wiki_index_strip_html(self):
     """HTML should be stripped, not indexed."""
     wc = WikiClient()
     results = wc.query('strong')
     eq_(0, len(results))

 def test_wiki_index_content(self):
     """Obviously the content should be indexed."""
     wc = WikiClient()
     results = wc.query('video')
     eq_(1, len(results))
     eq_(1, results[0]['id'])

 def test_indexer(self):
     wc = WikiClient()
     results = wc.query('audio')
     eq_(2, len(results))

 def test_range_filter(self):
     """Test filtering on a range."""
     wc = WikiClient()
     filter_ = ({"filter": "updated", "max": 1285765791, "min": 1284664176, "range": True},)
     results = wc.query("", filter_)
     eq_(2, len(results))

 def test_none_content_excerpt(self):
     """SearchClient.excerpt() returns empty string for None type."""
     wc = WikiClient()
     eq_('', wc.excerpt(None, 'test'))

 def test_clean_excerpt(self):
     """SearchClient.excerpt() should not allow disallowed HTML through."""
     wc = WikiClient()
     eq_('<b>test</b>&lt;/style&gt;', wc.excerpt('test</style>', 'test'))

def _search_suggestions(query, locale):
    """Return an iterable of the most relevant wiki pages and questions.

    query -- full text to search on
    locale -- locale to limit to

    Items are dicts of:
        {
            'type':
            'object':
        }

    Returns up to 3 wiki pages, then up to 3 questions.

    TODO: ZOMFG this needs to be refactored and the search app should
          provide an internal API. Seriously.

    """

    # Max number of search results per type.
    WIKI_RESULTS = QUESTIONS_RESULTS = 3

    # Search wiki pages:
    wiki_searcher = WikiClient()
    filters = [{'filter': 'locale',
                'value': (sphinx_locale(locale),)},
               {'filter': 'category',
                'value': [x for x in settings.SEARCH_DEFAULT_CATEGORIES
                          if x >= 0]},
               {'filter': 'category',
                'exclude': True,
                'value': [-x for x in settings.SEARCH_DEFAULT_CATEGORIES
                          if x < 0]}]
    raw_results = wiki_searcher.query(query, filters=filters,
                                      limit=WIKI_RESULTS)
    # Lazily build excerpts from results. Stop when we have enough:
    results = []
    for r in raw_results:
        try:
            doc = Document.objects.select_related('current_revision').\
                get(pk=r['id'])
            results.append({
                'type': 'document',
                'object': doc,
            })
        except Document.DoesNotExist:
            pass

    question_searcher = QuestionsClient()
    # questions app is en-US only.
    raw_results = question_searcher.query(query,
                                          limit=QUESTIONS_RESULTS)
    for r in raw_results:
        try:
            q = Question.objects.get(pk=r['attrs']['question_id'])
            results.append({
                'type': 'question',
                'object': q
            })
        except Question.DoesNotExist:
            pass

    return results

 def test_clean_excerpt(self):
     """SearchClient.excerpt() should not allow disallowed HTML through."""
     wc = WikiClient()  # Index strips HTML
     input = 'test <div>the start of something</div>'
     output_strip = '<b>test</b>  the start of something'
     eq_(output_strip, wc.excerpt(input, 'test'))