OwlBurst/Ponysearch
1
0
Fork 0
forked from Ponysearch/Ponysearch
Code Pull requests Activity

[fix] fix duden engine (#1594)

Browse source
This commit is contained in:
rachmadani haryono 2019-07-25 14:17:45 +08:00 committed by Alexandre Flament
parent 87baa74a86
commit 3b1122c5fa
2 changed files with 28 additions and 19 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
searx/engines/duden.py
View file

@ -11,7 +11,7 @@
from lxml import html, etree from lxml import html, etree
import re import re
from searx.engines.xpath import extract_text from searx.engines.xpath import extract_text
from searx.url_utils import quote from searx.url_utils import quote, urljoin
from searx import logger from searx import logger
categories = ['general'] categories = ['general']
@ -20,7 +20,7 @@ language_support = False
# search-url # search-url
base_url = 'https://www.duden.de/' base_url = 'https://www.duden.de/'
search_url = base_url + 'suchen/dudenonline/{query}?page={offset}' search_url = base_url + 'suchen/dudenonline/{query}?search_api_fulltext=&page={offset}'
def request(query, params): def request(query, params):
@ -35,6 +35,10 @@ def request(query, params):
''' '''
offset = (params['pageno'] - 1) offset = (params['pageno'] - 1)
if offset == 0:
search_url_fmt = base_url + 'suchen/dudenonline/{query}'
params['url'] = search_url_fmt.format(query=quote(query))
else:
params['url'] = search_url.format(offset=offset, query=quote(query)) params['url'] = search_url.format(offset=offset, query=quote(query))
return params return params
@ -58,12 +62,11 @@ def response(resp):
logger.debug("Couldn't read number of results.") logger.debug("Couldn't read number of results.")
pass pass
for result in dom.xpath('//section[@class="wide" and not(contains(@style,"overflow:hidden"))]'): for result in dom.xpath('//section[not(contains(@class, "essay"))]'):
try: try:
logger.debug("running for %s" % str(result)) url = result.xpath('.//h2/a')[0].get('href')
link = result.xpath('.//h2/a')[0] url = urljoin(base_url, url)
url = link.attrib.get('href') title = result.xpath('string(.//h2/a)').strip()
title = result.xpath('string(.//h2/a)')
content = extract_text(result.xpath('.//p')) content = extract_text(result.xpath('.//p'))
# append result # append result
results.append({'url': url, results.append({'url': url,

20
tests/unit/engines/test_duden.py
View file

@ -10,24 +10,30 @@ class TestDudenEngine(SearxTestCase):
def test_request(self): def test_request(self):
query = 'Haus' query = 'Haus'
dic = defaultdict(dict) dic = defaultdict(dict)
dic['pageno'] = 1 data = [
[1, 'https://www.duden.de/suchen/dudenonline/Haus'],
[2, 'https://www.duden.de/suchen/dudenonline/Haus?search_api_fulltext=&page=1']
]
for page_no, exp_res in data:
dic['pageno'] = page_no
params = duden.request(query, dic) params = duden.request(query, dic)
self.assertTrue('url' in params) self.assertTrue('url' in params)
self.assertTrue(query in params['url']) self.assertTrue(query in params['url'])
self.assertTrue('duden.de' in params['url']) self.assertTrue('duden.de' in params['url'])
self.assertEqual(params['url'], exp_res)
def test_response(self): def test_response(self):
resp = mock.Mock(text='<html></html>') resp = mock.Mock(text='<html></html>')
self.assertEqual(duden.response(resp), []) self.assertEqual(duden.response(resp), [])
html = """ html = """
<section class="wide"> <section class="vignette">
<h2><a href="https://this.is.the.url/" class="hidden-link"><strong>This is the title</strong> also here</a></h2> <h2"> <a href="/rechtschreibung/Haus">
<p>This is the <strong>content</strong></p> <strong>This is the title also here</strong>
<a href="https://this.is.the.url/">Zum vollst&auml;ndigen Artikel</a> </a> </h2>
<p>This is the content</p>
</section> </section>
""" """
resp = mock.Mock(text=html) resp = mock.Mock(text=html)
results = duden.response(resp) results = duden.response(resp)
@ -36,6 +42,6 @@ class TestDudenEngine(SearxTestCase):
# testing result (dictionary entry) # testing result (dictionary entry)
r = results[0] r = results[0]
self.assertEqual(r['url'], 'https://this.is.the.url/') self.assertEqual(r['url'], 'https://www.duden.de/rechtschreibung/Haus')
self.assertEqual(r['title'], 'This is the title also here') self.assertEqual(r['title'], 'This is the title also here')
self.assertEqual(r['content'], 'This is the content') self.assertEqual(r['content'], 'This is the content')