From 5ce309183788ebca1c501380bb72d73a02405d96 Mon Sep 17 00:00:00 2001 From: Martin Fischer Date: Tue, 18 Jan 2022 15:38:17 +0100 Subject: [PATCH 01/10] [enh] add pyrightconfig.json By adding this file Pyright automatically detects the packages from our Python virtual environment. This can be tested by using the Pyright extension for VS Code or by running npx pyright. --- pyrightconfig.json | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 pyrightconfig.json diff --git a/pyrightconfig.json b/pyrightconfig.json new file mode 100644 index 000000000..33e94f253 --- /dev/null +++ b/pyrightconfig.json @@ -0,0 +1,4 @@ +{ + "venvPath": "local", + "venv": "py3" +} From 87c8c5fd1bc7bf855d74ae37af947b292c262873 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Wed, 19 Jan 2022 10:08:24 +0100 Subject: [PATCH 02/10] [fix] pyrightconfig.json include only dedicated folders in the test Without specifying folders to check, pyright will also scan files in folders like ./build, ./cache, ./.nvm and more. Signed-off-by: Markus Heiser --- .gitignore | 2 +- pyrightconfig.json | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index efea6c872..bacf9d7a9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ -# to sync with .dockerignore +# to sync with .dockerignore & pyrightconfig.json *.pyc */*.pyc diff --git a/pyrightconfig.json b/pyrightconfig.json index 33e94f253..56573c75f 100644 --- a/pyrightconfig.json +++ b/pyrightconfig.json @@ -1,4 +1,9 @@ { "venvPath": "local", - "venv": "py3" + "venv": "py3", + "include": [ + "searx", + "searxng_extra", + "tests" + ] } From 5d9188c7e93f010f1d1d8d88d3b0791d54621b6c Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Wed, 19 Jan 2022 10:14:39 +0100 Subject: [PATCH 03/10] [mod] add pyright to nvm's node installation Pyright [1] is in the nvm enviroment, may be you need to rebuild the nvm environment and install nodejs in:: ./manage nvm.clean ./manage nvm.nodejs The last command installs nodejs and the packages from .nvm_packages. You can test your pyright installation, to get a bash within the nvm run:: ./manage nvm.bash (nvm) $ which pyright ./.nvm/versions/node/v16.13.0/bin/pyright If you have a local nvm in your HOME folder, the output from ``which`` is different. Press ``[CTRL-D]`` to get out of this bash. [1] https://github.com/microsoft/pyright Signed-off-by: Markus Heiser --- .nvm_packages | 1 + 1 file changed, 1 insertion(+) diff --git a/.nvm_packages b/.nvm_packages index 13eabc5c2..fe8585c3b 100644 --- a/.nvm_packages +++ b/.nvm_packages @@ -5,4 +5,5 @@ # [1] https://github.com/nvm-sh/nvm#default-global-packages-from-file-while-installing eslint +pyright From 834277321648297522ce2dbbf46559d96ca7d991 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Wed, 19 Jan 2022 10:31:45 +0100 Subject: [PATCH 04/10] [mod] manage test.pyright: a commandline to run pyright tests This patch implements the command and Makefile target:: ./manage test.pyright make test.pyright Signed-off-by: Markus Heiser --- Makefile | 2 +- manage | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 9d49e8d5f..3f46d109c 100644 --- a/Makefile +++ b/Makefile @@ -86,7 +86,7 @@ MANAGE += py.build py.clean MANAGE += pyenv pyenv.install pyenv.uninstall MANAGE += pypi.upload pypi.upload.test MANAGE += format.python -MANAGE += test.yamllint test.pylint test.black test.unit test.coverage test.robot test.rst test.clean +MANAGE += test.yamllint test.pylint test.pyright test.black test.unit test.coverage test.robot test.rst test.clean MANAGE += themes.all themes.oscar themes.simple themes.simple.test pygments.less MANAGE += static.build.commit static.build.drop static.build.restore MANAGE += nvm.install nvm.clean nvm.status nvm.nodejs diff --git a/manage b/manage index 1f4ffa53b..b70e81f52 100755 --- a/manage +++ b/manage @@ -103,6 +103,7 @@ format.: test.: yamllint : lint YAML files (YAMLLINT_FILES) pylint : lint PYLINT_FILES, searx/engines, searx & tests + pyright : static type check of python sources black : check black code format unit : run unit tests coverage : run unit tests with coverage @@ -682,6 +683,15 @@ test.pylint() { dump_return $? } + +test.pyright() { + build_msg TEST "[pyright] static type check of python sources" + nodejs.ensure + pyenv.cmd pyright + dump_return $? +} + + test.black() { build_msg TEST "[black] \$BLACK_TARGETS" pyenv.cmd black --check --diff "${BLACK_OPTIONS[@]}" "${BLACK_TARGETS[@]}" From aaf616fbd63f3b58fc88a2f60e04b9998ef1654a Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Wed, 19 Jan 2022 12:45:49 +0100 Subject: [PATCH 05/10] [emacs] .dir-locals.el: activate pyright in python-mode To get in use of pyright type cheker in Emacs, a pyright installation [1] is needed and in Emacs the lsp-pyright package has to be installed:: M-x package-install lsp-pyright [1] https://github.com/Microsoft/pyright [2] https://github.com/emacs-lsp/lsp-pyright Signed-off-by: Markus Heiser --- .dir-locals.el | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.dir-locals.el b/.dir-locals.el index 43d6fa551..419311ee7 100644 --- a/.dir-locals.el +++ b/.dir-locals.el @@ -32,6 +32,10 @@ ;; Jedi, flycheck & other python stuff should use the 'python-shell-interpreter' ;; from the local py3 environment. ;; +;; For pyright support you need to install:: +;; +;; M-x package-install lsp-pyright +;; ;; Other useful jedi stuff you might add to your ~/.emacs:: ;; ;; (global-set-key [f6] 'flycheck-mode) @@ -104,7 +108,10 @@ (python-mode . ((eval . (progn - + ;; use nodejs from the (local) NVM environment (see nvm-dir) + (nvm-use-for-buffer) + (if (featurep 'lsp-pyright) + (lsp)) (setq-local python-environment-virtualenv (list (expand-file-name "bin/virtualenv" python-shell-virtualenv-root) ;;"--system-site-packages" From 3cd5ce55e826a665c6113d5da8195aec9567631a Mon Sep 17 00:00:00 2001 From: Martin Fischer Date: Thu, 20 Jan 2022 07:04:39 +0100 Subject: [PATCH 06/10] [enh] clarify why we run pyright in the pyenv --- manage | 2 ++ 1 file changed, 2 insertions(+) diff --git a/manage b/manage index b70e81f52..0c0e851c7 100755 --- a/manage +++ b/manage @@ -687,6 +687,8 @@ test.pylint() { test.pyright() { build_msg TEST "[pyright] static type check of python sources" nodejs.ensure + # We run Pyright in the virtual environment because Pyright + # executes "python" to determine the Python version. pyenv.cmd pyright dump_return $? } From 96450b17d4304a740c0af934cbb841de67178d94 Mon Sep 17 00:00:00 2001 From: Martin Fischer Date: Thu, 20 Jan 2022 08:26:16 +0100 Subject: [PATCH 07/10] [mod] add test.pyright to test & ci.test targets Since we currently have many type checking errors, we for now only test with typeCheckingMode: off which makes pyright only check files that contain a comment: # pyright: basic to enable basic type checking, or # pyright: strict to enable strict type checking. --- Makefile | 4 ++-- manage | 2 +- pyrightconfig-ci.json | 10 ++++++++++ 3 files changed, 13 insertions(+), 3 deletions(-) create mode 100644 pyrightconfig-ci.json diff --git a/Makefile b/Makefile index 3f46d109c..0c0ec6c0a 100644 --- a/Makefile +++ b/Makefile @@ -50,8 +50,8 @@ search.checker.%: install $(Q)./manage pyenv.cmd searx-checker -v "$(subst _, ,$(patsubst search.checker.%,%,$@))" PHONY += test ci.test test.shell -ci.test: test.yamllint test.black test.pylint test.unit test.robot test.rst -test: test.yamllint test.black test.pylint test.unit test.robot test.rst test.shell +ci.test: test.yamllint test.black test.pyright test.pylint test.unit test.robot test.rst +test: test.yamllint test.black test.pyright test.pylint test.unit test.robot test.rst test.shell test.shell: $(Q)shellcheck -x -s dash \ dockerfiles/docker-entrypoint.sh diff --git a/manage b/manage index 0c0e851c7..aec3906df 100755 --- a/manage +++ b/manage @@ -689,7 +689,7 @@ test.pyright() { nodejs.ensure # We run Pyright in the virtual environment because Pyright # executes "python" to determine the Python version. - pyenv.cmd pyright + pyenv.cmd pyright -p pyrightconfig-ci.json dump_return $? } diff --git a/pyrightconfig-ci.json b/pyrightconfig-ci.json new file mode 100644 index 000000000..9082b0790 --- /dev/null +++ b/pyrightconfig-ci.json @@ -0,0 +1,10 @@ +{ + "venvPath": "local", + "venv": "py3", + "include": [ + "searx", + "searxng_extra", + "tests" + ], + "typeCheckingMode": "off" +} From 6d701d2fea07198807351c7c4856001c0f1ebffb Mon Sep 17 00:00:00 2001 From: Martin Fischer Date: Thu, 20 Jan 2022 09:11:37 +0100 Subject: [PATCH 08/10] [fix] ensure that test.pyright installs pyright nodejs.ensure only sets up NVM if there isn't a system-wide installation of Node that matches our NODE_MINIMUM_VERSION. The ubuntu image in the CI comes with an up to date node version, so pyright from .nvm_packages is never installed. This commit fixes this by introducing a package.json file. --- .gitignore | 3 +++ .nvm_packages | 1 - manage | 3 ++- package.json | 5 +++++ 4 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 package.json diff --git a/.gitignore b/.gitignore index bacf9d7a9..9b3eff158 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,6 @@ dist/ local/ gh-pages/ *.egg-info/ + +/package-lock.json +/node_modules/ diff --git a/.nvm_packages b/.nvm_packages index fe8585c3b..13eabc5c2 100644 --- a/.nvm_packages +++ b/.nvm_packages @@ -5,5 +5,4 @@ # [1] https://github.com/nvm-sh/nvm#default-global-packages-from-file-while-installing eslint -pyright diff --git a/manage b/manage index aec3906df..45853de3d 100755 --- a/manage +++ b/manage @@ -687,9 +687,10 @@ test.pylint() { test.pyright() { build_msg TEST "[pyright] static type check of python sources" nodejs.ensure + npm install # We run Pyright in the virtual environment because Pyright # executes "python" to determine the Python version. - pyenv.cmd pyright -p pyrightconfig-ci.json + pyenv.cmd npx --no-install pyright -p pyrightconfig-ci.json dump_return $? } diff --git a/package.json b/package.json new file mode 100644 index 000000000..187d17a4b --- /dev/null +++ b/package.json @@ -0,0 +1,5 @@ +{ + "dependencies": { + "pyright": "^1.1.212" + } +} From cac03529860e0fd19585b35a66f5de4f0ea0559d Mon Sep 17 00:00:00 2001 From: Martin Fischer Date: Fri, 21 Jan 2022 17:34:25 +0100 Subject: [PATCH 09/10] [mod] remove .nvm_packages, add eslint to package.json --- .nvm_packages | 8 -------- package.json | 1 + utils/lib_nvm.sh | 1 - 3 files changed, 1 insertion(+), 9 deletions(-) delete mode 100644 .nvm_packages diff --git a/.nvm_packages b/.nvm_packages deleted file mode 100644 index 13eabc5c2..000000000 --- a/.nvm_packages +++ /dev/null @@ -1,8 +0,0 @@ -# -*- coding: utf-8; mode: conf-unix -*- -# -# Developement tools pre-installed in NVM's node installation [1] -# -# [1] https://github.com/nvm-sh/nvm#default-global-packages-from-file-while-installing - -eslint - diff --git a/package.json b/package.json index 187d17a4b..e65e0585d 100644 --- a/package.json +++ b/package.json @@ -1,5 +1,6 @@ { "dependencies": { + "eslint": "^8.7.0", "pyright": "^1.1.212" } } diff --git a/utils/lib_nvm.sh b/utils/lib_nvm.sh index 267ba8a4a..cd2fbc189 100755 --- a/utils/lib_nvm.sh +++ b/utils/lib_nvm.sh @@ -117,7 +117,6 @@ nvm.install() { info_msg "checkout ${NVM_VERSION_TAG}" git checkout "${NVM_VERSION_TAG}" 2>&1 | prefix_stdout " ${_Yellow}||${_creset} " popd &> /dev/null - cp "${REPO_ROOT}/.nvm_packages" "${NVM_DIR}/default-packages" nvm.env } From 309147d86f08f69ce1467aa4d25d3d6a56b2bc46 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Sat, 22 Jan 2022 09:26:17 +0100 Subject: [PATCH 10/10] [mod] introduce node.env.devtools function --- manage | 12 +++++++++--- utils/lib_nvm.sh | 3 +++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/manage b/manage index 45853de3d..6cc98c1be 100755 --- a/manage +++ b/manage @@ -20,6 +20,8 @@ source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_go.sh" # shellcheck source=utils/lib_redis.sh source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_redis.sh" +PATH="${REPO_ROOT}/node_modules/.bin:${PATH}" + # config PYOBJECTS="searx" @@ -559,6 +561,12 @@ node.env() { dump_return $? } +node.env.devtools() { + nodejs.ensure + build_msg INSTALL "package.json: developer and CI tools" + npm install +} + node.clean() { if ! required_commands npm 2>/dev/null; then build_msg CLEAN "npm is not installed / ignore npm dependencies" @@ -683,11 +691,9 @@ test.pylint() { dump_return $? } - test.pyright() { build_msg TEST "[pyright] static type check of python sources" - nodejs.ensure - npm install + node.env.devtools # We run Pyright in the virtual environment because Pyright # executes "python" to determine the Python version. pyenv.cmd npx --no-install pyright -p pyrightconfig-ci.json diff --git a/utils/lib_nvm.sh b/utils/lib_nvm.sh index cd2fbc189..ec0ed5c8f 100755 --- a/utils/lib_nvm.sh +++ b/utils/lib_nvm.sh @@ -117,6 +117,9 @@ nvm.install() { info_msg "checkout ${NVM_VERSION_TAG}" git checkout "${NVM_VERSION_TAG}" 2>&1 | prefix_stdout " ${_Yellow}||${_creset} " popd &> /dev/null + if [ -f "${REPO_ROOT}/.nvm_packages" ]; then + cp "${REPO_ROOT}/.nvm_packages" "${NVM_DIR}/default-packages" + fi nvm.env }