OwlBurst/Ponysearch
1
0
Fork 0
forked from Ponysearch/Ponysearch
Code Pull requests Activity

[fix] ddg engines (get_vqd) - the vqd value is no longer in the form

Browse source 
Closes: https://github.com/searxng/searxng/issues/3276
Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
This commit is contained in:
Markus Heiser 2024-03-03 19:00:02 +01:00 committed by Markus Heiser
parent b7be53bb90
commit d97b84bea2
1 changed files with 14 additions and 24 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

38
searx/engines/duckduckgo.py
View file

@ -65,7 +65,7 @@ def cache_vqd(query, value):
c = redisdb.client() c = redisdb.client()
if c: if c:
logger.debug("cache vqd value: %s", value) logger.debug("cache vqd value: %s", value)
key = 'SearXNG_ddg_vqd' + redislib.secret_hash(query) key = 'SearXNG_ddg_web_vqd' + redislib.secret_hash(query)
c.set(key, value, ex=600) c.set(key, value, ex=600)
@ -105,27 +105,25 @@ def get_vqd(query):
- DuckDuckGo News: ``https://duckduckgo.com/news.js??q=...&vqd=...`` - DuckDuckGo News: ``https://duckduckgo.com/news.js??q=...&vqd=...``
""" """
value = '' value = None
c = redisdb.client() c = redisdb.client()
if c: if c:
key = 'SearXNG_ddg_vqd' + redislib.secret_hash(query) key = 'SearXNG_ddg_web_vqd' + redislib.secret_hash(query)
value = c.get(key) value = c.get(key)
if value or value == b'': if value or value == b'':
value = value.decode('utf-8') value = value.decode('utf-8')
logger.debug("re-use cached vqd value: %s", value) logger.debug("re-use cached vqd value: %s", value)
return value return value
query_url = 'https://lite.duckduckgo.com/lite/?{args}'.format(args=urlencode({'q': query})) query_url = 'https://duckduckgo.com/?' + urlencode({'q': query})
res = get(query_url) res = get(query_url)
doc = lxml.html.fromstring(res.text) doc = lxml.html.fromstring(res.text)
value = doc.xpath("//input[@name='vqd']/@value") for script in doc.xpath("//script[@type='text/javascript']"):
if value: script = script.text
value = value[0] if 'vqd="' in script:
else: value = script[script.index('vqd="') + 5 :]
# Some search terms do not have results and therefore no vqd value. If value = value[: value.index('"')]
# no vqd value can be determined for the search term, an empty string is break
# chached.
value = ''
logger.debug("new vqd value: '%s'", value) logger.debug("new vqd value: '%s'", value)
cache_vqd(query, value) cache_vqd(query, value)
return value return value
@ -228,10 +226,6 @@ def request(query, params):
# request needs a vqd argument # request needs a vqd argument
vqd = get_vqd(query) vqd = get_vqd(query)
if not vqd:
# some search terms do not have results and therefore no vqd value
params['url'] = None
return params
# quote ddg bangs # quote ddg bangs
query_parts = [] query_parts = []
@ -260,14 +254,14 @@ def request(query, params):
# initial page does not have an offset # initial page does not have an offset
if params['pageno'] == 2: if params['pageno'] == 2:
# second page does have an offset of 30 # second page does have an offset of 20
offset = (params['pageno'] - 1) * 30 offset = (params['pageno'] - 1) * 20
params['data']['s'] = offset params['data']['s'] = offset
params['data']['dc'] = offset + 1 params['data']['dc'] = offset + 1
elif params['pageno'] > 2: elif params['pageno'] > 2:
# third and following pages do have an offset of 30 + n*50 # third and following pages do have an offset of 20 + n*50
offset = 30 + (params['pageno'] - 2) * 50 offset = 20 + (params['pageno'] - 2) * 50
params['data']['s'] = offset params['data']['s'] = offset
params['data']['dc'] = offset + 1 params['data']['dc'] = offset + 1
@ -322,10 +316,6 @@ def response(resp):
form_data['o'] = eval_xpath(form, '//input[@name="o"]/@value')[0] form_data['o'] = eval_xpath(form, '//input[@name="o"]/@value')[0]
logger.debug('form_data: %s', form_data) logger.debug('form_data: %s', form_data)
value = eval_xpath(form, '//input[@name="vqd"]/@value')[0]
query = resp.search_params['data']['q']
cache_vqd(query, value)
tr_rows = eval_xpath(result_table, './/tr') tr_rows = eval_xpath(result_table, './/tr')
# In the last <tr> is the form of the 'previous/next page' links # In the last <tr> is the form of the 'previous/next page' links
tr_rows = tr_rows[:-1] tr_rows = tr_rows[:-1]