diff --git a/.dir-locals.el b/.dir-locals.el index 47a06f4cb..1e319357a 100644 --- a/.dir-locals.el +++ b/.dir-locals.el @@ -3,18 +3,23 @@ ;; Per-Directory Local Variables: ;; https://www.gnu.org/software/emacs/manual/html_node/emacs/Directory-Variables.html ;; -;; .. hint:: +;; For full fledge developer tools install emacs packages: ;; -;; If you get ``*** EPC Error ***`` (even after a jedi:install-server) in -;; your emacs session, mostly you have jedi-mode enabled but the python -;; environment is missed. The python environment has to be next to the -;; ``/.dir-locals.el`` in:: +;; M-x package-install ... ;; -;; ./local/py3 +;; magit gitconfig +;; nvm lsp-mode lsp-pyright lsp-eslint +;; pyvenv pylint pip-requirements +;; jinja2-mode +;; json-mode +;; company company-jedi company-quickhelp company-shell +;; realgud +;; sphinx-doc markdown-mode graphviz-dot-mode +;; apache-mode nginx-mode ;; -;; To setup such an environment, build target:: +;; To setup a developer environment, build target:: ;; -;; $ make pyenv.install +;; $ make node.env.dev pyenv.install ;; ;; Some buffer locals are referencing the project environment: ;; @@ -29,26 +34,11 @@ ;; (setq python-shell-virtualenv-root "/path/to/env/") ;; - python-shell-interpreter --> /local/py3/bin/python ;; +;; Python development: +;; ;; Jedi, flycheck & other python stuff should use the 'python-shell-interpreter' ;; from the local py3 environment. ;; -;; For pyright support you need to install:: -;; -;; M-x package-install lsp-pyright -;; -;; Other useful jedi stuff you might add to your ~/.emacs:: -;; -;; (global-set-key [f6] 'flycheck-mode) -;; (add-hook 'python-mode-hook 'my:python-mode-hook) -;; -;; (defun my:python-mode-hook () -;; (add-to-list 'company-backends 'company-jedi) -;; (require 'jedi-core) -;; (jedi:setup) -;; (define-key python-mode-map (kbd "C-c C-d") 'jedi:show-doc) -;; (define-key python-mode-map (kbd "M-.") 'jedi:goto-definition) -;; (define-key python-mode-map (kbd "M-,") 'jedi:goto-definition-pop-marker) -;; ) ((nil . ((fill-column . 80) @@ -67,6 +57,12 @@ ;; to get in use of NVM environment, install https://github.com/rejeep/nvm.el (setq-local nvm-dir (expand-file-name "./.nvm" prj-root)) + ;; use nodejs from the (local) NVM environment (see nvm-dir) + (nvm-use-for-buffer) + (ignore-errors (require 'lsp)) + (setq-local lsp-server-install-dir (car (cdr nvm-current-version))) + (setq-local lsp-enable-file-watchers nil) + ;; use 'py3' environment as default (setq-local python-environment-default-root-name "py3") @@ -100,22 +96,22 @@ (js-mode . ((eval . (progn - ;; use nodejs from the (local) NVM environment (see nvm-dir) - (nvm-use-for-buffer) + (ignore-errors (require 'lsp-eslint)) (setq-local js-indent-level 2) ;; flycheck should use the eslint checker from developer tools (setq-local flycheck-javascript-eslint-executable (expand-file-name "node_modules/.bin/eslint" prj-root)) + ;; (flycheck-mode) - (flycheck-mode) + (if (featurep 'lsp-eslint) + (lsp)) )))) (python-mode . ((eval . (progn - ;; use nodejs from the (local) NVM environment (see nvm-dir) - (nvm-use-for-buffer) - (if (featurep 'lsp-pyright) - (lsp)) + (ignore-errors (require 'jedi-core)) + (ignore-errors (require 'lsp-pyright)) + (ignore-errors (sphinx-doc-mode)) (setq-local python-environment-virtualenv (list (expand-file-name "bin/virtualenv" python-shell-virtualenv-root) ;;"--system-site-packages" @@ -124,6 +120,9 @@ (setq-local pylint-command (expand-file-name "bin/pylint" python-shell-virtualenv-root)) + (if (featurep 'lsp-pyright) + (lsp)) + ;; pylint will find the '.pylintrc' file next to the CWD ;; https://pylint.readthedocs.io/en/latest/user_guide/run.html#command-line-options (setq-local flycheck-pylintrc diff --git a/.pylintrc b/.pylintrc index 51cea773e..a5b00227e 100644 --- a/.pylintrc +++ b/.pylintrc @@ -404,4 +404,4 @@ known-third-party=enchant # Exceptions that will emit a warning when being caught. Defaults to # "Exception" -overgeneral-exceptions=Exception +overgeneral-exceptions=builtins.Exception diff --git a/AUTHORS.rst b/AUTHORS.rst index 75bc9f87b..f6c04a35f 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -168,3 +168,4 @@ features or generally made searx better: - Ahmad Alkadri ``_ - Milad Laly @Milad-Laly - @llmII +- @blob42 ``_ diff --git a/Makefile b/Makefile index 66c644ba2..31b3787a8 100644 --- a/Makefile +++ b/Makefile @@ -80,7 +80,7 @@ MANAGE += data.all data.languages data.useragents data.osm_keys_tags MANAGE += docs.html docs.live docs.gh-pages docs.prebuild docs.clean MANAGE += docker.build docker.push docker.buildx MANAGE += gecko.driver -MANAGE += node.env node.clean +MANAGE += node.env node.env.dev node.clean MANAGE += py.build py.clean MANAGE += pyenv pyenv.install pyenv.uninstall MANAGE += pypi.upload pypi.upload.test diff --git a/docs/admin/engines/settings.rst b/docs/admin/engines/settings.rst index 97f5ef63e..e8d00ec1f 100644 --- a/docs/admin/engines/settings.rst +++ b/docs/admin/engines/settings.rst @@ -657,8 +657,9 @@ and can relied on the default configuration :origin:`searx/settings.yml` using: ``engines:`` With ``use_default_settings: true``, each settings can be override in a similar way, the ``engines`` section is merged according to the engine - ``name``. In this example, SearXNG will load all the engine and the arch linux - wiki engine has a :ref:`token `: + ``name``. In this example, SearXNG will load all the default engines, will + enable the ``bing`` engine and define a :ref:`token ` for + the arch linux engine: .. code-block:: yaml @@ -668,6 +669,9 @@ and can relied on the default configuration :origin:`searx/settings.yml` using: engines: - name: arch linux wiki tokens: ['$ecretValue'] + - name: bing + disabled: false + ``engines:`` / ``remove:`` It is possible to remove some engines from the default settings. The following diff --git a/manage b/manage index ce72364de..ad851752b 100755 --- a/manage +++ b/manage @@ -97,7 +97,8 @@ redis: install : create user (${REDIS_USER}) and install systemd service (${REDIS_SERVICE_NAME}) help : show more redis commands node.: - env : download & install npm dependencies locally + env : download & install SearXNG's npm dependencies locally + env.dev : download & install developer and CI tools clean : drop locally npm installations py.: build : Build python packages at ./${PYDIST} @@ -563,16 +564,15 @@ nodejs.ensure() { node.env() { nodejs.ensure ( set -e - - build_msg INSTALL "searx/static/themes/simple/package.json" + build_msg INSTALL "./searx/static/themes/simple/package.json" npm --prefix searx/static/themes/simple install ) dump_return $? } -node.env.devtools() { +node.env.dev() { nodejs.ensure - build_msg INSTALL "package.json: developer and CI tools" + build_msg INSTALL "./package.json: developer and CI tools" npm install } @@ -585,6 +585,10 @@ node.clean() { ( set -e npm --prefix searx/static/themes/simple run clean ) + build_msg CLEAN "locally installed developer and CI tools" + ( set -e + npm --prefix . run clean + ) dump_return $? } @@ -702,7 +706,7 @@ test.pylint() { test.pyright() { build_msg TEST "[pyright] static type check of python sources" - node.env.devtools + node.env.dev # We run Pyright in the virtual environment because Pyright # executes "python" to determine the Python version. build_msg TEST "[pyright] suppress warnings related to intentional monkey patching" diff --git a/package.json b/package.json index 9c4b0186c..5eb3da891 100644 --- a/package.json +++ b/package.json @@ -2,5 +2,8 @@ "dependencies": { "eslint": "^8.18.0", "pyright": "^1.1.255" + }, + "scripts": { + "clean": "rm -Rf node_modules package-lock.json" } } diff --git a/requirements-dev.txt b/requirements-dev.txt index 0c4b07311..df395af59 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,7 +2,7 @@ mock==5.0.1 nose2[coverage_plugin]==0.12.0 cov-core==1.15.0 black==22.12.0 -pylint==2.15.10 +pylint==2.16.2 splinter==0.19.0 selenium==4.8.0 twine==4.0.2 diff --git a/requirements.txt b/requirements.txt index f30f43716..80f9257cb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ certifi==2022.12.7 babel==2.11.0 flask-babel==3.0.1 -flask==2.2.2 +flask==2.2.3 jinja2==3.1.2 lxml==4.9.2 pygments==2.14.0 @@ -12,7 +12,7 @@ Brotli==1.0.9 uvloop==0.17.0 httpx-socks[asyncio]==0.7.2 setproctitle==1.3.2 -redis==4.4.2 +redis==4.5.1 markdown-it-py==2.1.0 -typing_extensions==4.4.0 +typing_extensions==4.5.0 fasttext-predict==0.9.2.1 diff --git a/searx/autocomplete.py b/searx/autocomplete.py index 6fb5537a2..aeb697a14 100644 --- a/searx/autocomplete.py +++ b/searx/autocomplete.py @@ -3,6 +3,7 @@ """This module implements functions needed for the autocompleter. """ +# pylint: disable=use-dict-literal from json import loads from urllib.parse import urlencode diff --git a/searx/engines/crossref.py b/searx/engines/crossref.py index fbe2f0c2a..e12a0da5b 100644 --- a/searx/engines/crossref.py +++ b/searx/engines/crossref.py @@ -2,6 +2,7 @@ # lint: pylint """Semantic Scholar (Science) """ +# pylint: disable=use-dict-literal from urllib.parse import urlencode from searx.utils import html_to_text diff --git a/searx/engines/docker_hub.py b/searx/engines/docker_hub.py index 1e492b196..cde96d016 100644 --- a/searx/engines/docker_hub.py +++ b/searx/engines/docker_hub.py @@ -3,6 +3,7 @@ """Docker Hub (IT) """ +# pylint: disable=use-dict-literal from json import loads from urllib.parse import urlencode diff --git a/searx/engines/gigablast.py b/searx/engines/gigablast.py index 1c40ff331..382711b10 100644 --- a/searx/engines/gigablast.py +++ b/searx/engines/gigablast.py @@ -3,7 +3,7 @@ """ Gigablast (Web) """ -# pylint: disable=invalid-name +# pylint: disable=invalid-name, use-dict-literal import re from time import time diff --git a/searx/plugins/tor_check.py b/searx/plugins/tor_check.py index 7d50bbcb5..831c90ce5 100644 --- a/searx/plugins/tor_check.py +++ b/searx/plugins/tor_check.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: AGPL-3.0-or-later # lint: pylint -"""A plugin to check if the ip address of the request is a TOR exit node if the +"""A plugin to check if the ip address of the request is a Tor exit-node if the user searches for ``tor-check``. It fetches the tor exit node list from https://check.torproject.org/exit-addresses and parses all the IPs into a list, then checks if the user's IP address is in it. @@ -26,8 +26,8 @@ name = gettext("Tor check plugin") '''Translated name of the plugin''' description = gettext( - "This plugin checks if the address of the request is a TOR exit node, and" - " informs the user if it is, like check.torproject.org but from searxng." + "This plugin checks if the address of the request is a Tor exit-node, and" + " informs the user if it is; like check.torproject.org, but from SearXNG." ) '''Translated description of the plugin.''' @@ -60,7 +60,7 @@ def post_search(request, search): # No answer, return error search.result_container.answers["tor"] = { "answer": gettext( - "The TOR exit node list (https://check.torproject.org/exit-addresses) is unreachable." + "Could not download the list of Tor exit-nodes from: https://check.torproject.org/exit-addresses" ) } return True @@ -75,13 +75,17 @@ def post_search(request, search): if ip_address in node_list: search.result_container.answers["tor"] = { "answer": gettext( - "You are using TOR. Your IP address seems to be: {ip_address}.".format(ip_address=ip_address) + "You are using Tor and it looks like you have this external IP address: {ip_address}".format( + ip_address=ip_address + ) ) } else: search.result_container.answers["tor"] = { "answer": gettext( - "You are not using TOR. Your IP address seems to be: {ip_address}.".format(ip_address=ip_address) + "You are not using Tor and you have this external IP address: {ip_address}".format( + ip_address=ip_address + ) ) } diff --git a/searx/search/processors/online.py b/searx/search/processors/online.py index dd5d1e36b..242718416 100644 --- a/searx/search/processors/online.py +++ b/searx/search/processors/online.py @@ -4,6 +4,7 @@ """Processores for engine-type: ``online`` """ +# pylint: disable=use-dict-literal from timeit import default_timer import asyncio diff --git a/searx/settings.yml b/searx/settings.yml index 4d91250bf..2f9c7bf12 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -583,6 +583,7 @@ engines: engine: tineye shortcut: tin timeout: 9.0 + disabled: true - name: etymonline engine: xpath @@ -1002,38 +1003,6 @@ engines: # collection: 'reviews' # name of the db collection # key: 'name' # key in the collection to search for - - name: neeva - engine: xpath - shortcut: nv - time_range_support: true - time_range_url: '&alf%5Bfreshness%5D={time_range_val}' - time_range_map: - day: 'Day' - week: 'Week' - month: 'Month' - year: 'Year' - search_url: https://neeva.com/search?q={query}&c=All&src=Pagination&page={pageno}{time_range} - results_xpath: //div[@class="web-index__component-2rKiM"] | //li[@class="web-rich-deep-links__deepLink-SIbD4"] - url_xpath: .//a[@class="lib-doc-title__link-1b9rC"]/@href | ./h2/a/@href - title_xpath: .//a[@class="lib-doc-title__link-1b9rC"] | ./h2/a - content_xpath: > - .//div[@class="lib-doc-snippet__component-3ewW6"]/text() | - .//div[@class="lib-doc-snippet__component-3ewW6"]/*[not(self::a)] | - ./p - content_html_to_text: true - suggestion_xpath: //span[@class="result-related-searches__link-2ho_u"] - paging: true - disabled: true - categories: [general, web] - timeout: 5.0 - soft_max_redirects: 2 - about: - website: https://neeva.com - official_api_documentation: - use_official_api: false - require_api_key: false - results: HTML - - name: npm engine: json_engine paging: true @@ -1523,7 +1492,8 @@ engines: - name: wiby engine: json_engine - search_url: https://wiby.me/json/?q={query} + paging: true + search_url: https://wiby.me/json/?q={query}&p={pageno} url_query: URL title_query: Title content_query: Snippet @@ -1908,13 +1878,15 @@ engines: engine: xpath paging: true search_url: https://petalsearch.com/search?query={query}&pn={pageno} - results_xpath: //div[@class="webpage-content"]/div[@class="title-cont"]/a - url_xpath: ./@href - title_xpath: . - content_xpath: ../../div[@class="webpage-text"] - suggestion_xpath: //div[@class="related-search-items"]/a + url_xpath: //div[@class='card-source'] + title_xpath: //div[@class='title-name'] + content_xpath: //div[@class='webpage-text'] first_page_num: 1 disabled: true + headers: + User-Agent: Mozilla/5.0 (Linux; Android 7.0;) \ + AppleWebKit/537.36 (KHTML, like Gecko) \ + Mobile Safari/537.36 (compatible; PetalBot;+https://webmaster.petalsearch.com/site/petalbot) about: website: https://petalsearch.com/ wikidata_id: Q104399280 @@ -1928,26 +1900,6 @@ engines: disabled: true timeout: 3.0 - - name: petalsearch news - shortcut: ptsn - categories: news - engine: xpath - paging: true - search_url: https://petalsearch.com/search?channel=news&query={query}&pn={pageno} - results_xpath: //div[@class="news-container"]/div/div/div/a - url_xpath: ./@href - title_xpath: ./div - content_xpath: ../div[@class="news-text"] - thumbnail_xpath: ../../../../img/@src - first_page_num: 1 - disabled: true - about: - website: https://petalsearch.com/ - wikidata_id: Q104399280 - use_official_api: false - require_api_key: false - results: HTML - - name: lib.rs shortcut: lrs engine: xpath diff --git a/searx/templates/simple/results.html b/searx/templates/simple/results.html index 755610e5b..f73a25d09 100644 --- a/searx/templates/simple/results.html +++ b/searx/templates/simple/results.html @@ -65,7 +65,7 @@ {% if suggestions %}