OwlBurst/Ponysearch
1
0
Fork 0
forked from Ponysearch/Ponysearch
Code Pull requests Activity

[mod] engine ask.com - parse JS result to JSON

Browse source 
Parse the result list from ask.com given in the variable named
window.MESON.initialState::

    <script nonce="..">
        window.MESON = window.MESON || {};
        window.MESON.initialState = {"siteConfig": ...
          ...}};
        window.MESON.loadedLang = "en";
    </script>

The result list is in field::

    json_resp['search']['webResults']['results']

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
This commit is contained in:
Markus Heiser 2024-02-16 18:17:03 +01:00 committed by Markus Heiser
parent 3df53d6e50
commit 76845ea42c
1 changed files with 33 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

48
searx/engines/ask.py
View file

@ -3,8 +3,9 @@
"""Ask.com"""
from urllib.parse import urlencode
import re
import dateutil
from lxml import html
from searx import utils
# Metadata
about = {
@ -37,20 +38,37 @@ def request(query, params):
def response(resp):
text = html.fromstring(resp.text).text_content()
urls_match = re.findall(r'"url":"(.*?)"', text)
titles_match = re.findall(r'"title":"(.*?)"', text)[3:]
content_match = re.findall(r'"abstract":"(.*?)"', text)
start_tag = 'window.MESON.initialState = {'
end_tag = '}};'
results = [
{
"url": url,
"title": title,
"content": content,
}
for url, title, content in zip(urls_match, titles_match, content_match)
if "&qo=relatedSearchNarrow" not in url
# Related searches shouldn't be in the search results: www.ask.com/web&q=related
]
dom = html.fromstring(resp.text)
script = utils.eval_xpath_getindex(dom, '//script', 0, default=None).text
pos = script.index(start_tag) + len(start_tag) - 1
script = script[pos:]
pos = script.index(end_tag) + len(end_tag) - 1
script = script[:pos]
json_resp = utils.js_variable_to_python(script)
results = []
for item in json_resp['search']['webResults']['results']:
pubdate_original = item.get('pubdate_original')
if pubdate_original:
pubdate_original = dateutil.parser.parse(pubdate_original)
metadata = [item.get(field) for field in ['category_l1', 'catsy'] if item.get(field)]
results.append(
{
"url": item['url'],
"title": item['title'],
"content": item['abstract'],
"publishedDate": pubdate_original,
# "img_src": item.get('image_url') or None, # these are not thumbs / to large
"metadata": ' | '.join(metadata),
}
)
return results