OwlBurst/Ponysearch
1
0
Fork 0
forked from Ponysearch/Ponysearch
Code Pull requests Activity

[fix] update_external_bangs: BANGS_URL 'https://duckduckgo.com/bang.js'

Browse source 
JSON file which contains the bangs / there is no longer a versioning of this
file.

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
This commit is contained in:
Markus Heiser 2024-03-10 11:49:37 +01:00 committed by Markus Heiser
parent e97e1f9110
commit cff0097289
1 changed files with 22 additions and 38 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

60
searxng_extra/update/update_external_bangs.py
View file

@ -2,50 +2,42 @@
# lint: pylint # lint: pylint
# SPDX-License-Identifier: AGPL-3.0-or-later # SPDX-License-Identifier: AGPL-3.0-or-later
"""Update :origin:`searx/data/external_bangs.json` using the duckduckgo bangs """Update :origin:`searx/data/external_bangs.json` using the duckduckgo bangs
(:origin:`CI Update data ... <.github/workflows/data-update.yml>`). from :py:obj:`BANGS_URL`.
https://duckduckgo.com/newbang loads: - :origin:`CI Update data ... <.github/workflows/data-update.yml>`
* a javascript which provides the bang version ( https://duckduckgo.com/bv1.js )
* a JSON file which contains the bangs ( https://duckduckgo.com/bang.v260.js for example )
This script loads the javascript, then the bangs.
The javascript URL may change in the future ( for example
https://duckduckgo.com/bv2.js ), but most probably it will requires to update
RE_BANG_VERSION
""" """
# pylint: disable=C0116
from pathlib import Path
import json import json
import re
from os.path import join
import httpx import httpx
from searx import searx_dir # pylint: disable=E0401 C0413 from searx import searx_dir
from searx.external_bang import LEAF_KEY from searx.external_bang import LEAF_KEY
# from https://duckduckgo.com/newbang
URL_BV1 = 'https://duckduckgo.com/bv1.js' BANGS_URL = 'https://duckduckgo.com/bang.js'
RE_BANG_VERSION = re.compile(r'\/bang\.v([0-9]+)\.js') """JSON file which contains the bangs."""
BANGS_DATA_FILE = Path(searx_dir) / 'data' / 'external_bangs.json'
HTTPS_COLON = 'https:' HTTPS_COLON = 'https:'
HTTP_COLON = 'http:' HTTP_COLON = 'http:'
def get_bang_url(): def main():
response = httpx.get(URL_BV1) print(f'fetch bangs from {BANGS_URL}')
response = httpx.get(BANGS_URL)
response.raise_for_status() response.raise_for_status()
ddg_bangs = json.loads(response.content.decode())
r = RE_BANG_VERSION.findall(response.text) trie = parse_ddg_bangs(ddg_bangs)
return f'https://duckduckgo.com/bang.v{r[0]}.js', r[0] output = {
'version': 0,
'trie': trie,
def fetch_ddg_bangs(url): }
response = httpx.get(url) with open(BANGS_DATA_FILE, 'w', encoding="utf8") as f:
response.raise_for_status() json.dump(output, f, sort_keys=True, ensure_ascii=False, indent=4)
return json.loads(response.content.decode())
def merge_when_no_leaf(node): def merge_when_no_leaf(node):
@ -151,13 +143,5 @@ def parse_ddg_bangs(ddg_bangs):
return bang_trie return bang_trie
def get_bangs_filename():
return join(join(searx_dir, "data"), "external_bangs.json")
if __name__ == '__main__': if __name__ == '__main__':
bangs_url, bangs_version = get_bang_url() main()
print(f'fetch bangs from {bangs_url}')
output = {'version': bangs_version, 'trie': parse_ddg_bangs(fetch_ddg_bangs(bangs_url))}
with open(get_bangs_filename(), 'w', encoding="utf8") as fp:
json.dump(output, fp, ensure_ascii=False, indent=4)