Merge pull request #506 from return42/nvm

[mod] Tools to install and maintain NVM versions manager for Node.js

.gitignore

@@ -9,6 +9,7 @@ geckodriver.log
 .coverage
 coverage/
 
+.nvm/
 cache/
 build/
 dist/

Makefile

@@ -36,7 +36,7 @@ install uninstall:
 	$(Q)./manage pyenv.$@
 
 PHONY += clean
-clean: py.clean docs.clean node.clean test.clean
+clean: py.clean docs.clean node.clean nvm.clean test.clean
 	$(Q)./manage build_msg CLEAN  "common files"
 	$(Q)find . -name '*.orig' -exec rm -f {} +
 	$(Q)find . -name '*.rej' -exec rm -f {} +
@@ -64,6 +64,7 @@ test.shell:
 		$(MTOOLS) \
 		utils/lib.sh \
 		utils/lib_install.sh \
+		utils/lib_nvm.sh \
 		utils/lib_static.sh \
 	        utils/filtron.sh \
 	        utils/searx.sh \
@@ -89,13 +90,13 @@ MANAGE += pypi.upload pypi.upload.test
 MANAGE += test.yamllint test.pylint test.pep8 test.unit test.coverage test.robot test.clean
 MANAGE += themes.all themes.oscar themes.simple pygments.less
 MANAGE += static.build.commit static.build.drop static.build.restore
+MANAGE += nvm.install nvm.clean nvm.status nvm.nodejs
 
 PHONY += $(MANAGE)
 
 $(MANAGE):
 	$(Q)$(MTOOLS) $@
 
-
 # deprecated
 
 PHONY += docs docs-clean docs-live docker themes

docs/conf.py

@@ -43,7 +43,10 @@ searx.engines.load_engines(searx.settings['engines'])
 jinja_contexts = {
     'searx': {
         'engines': searx.engines.engines,
-        'plugins': searx.plugins.plugins
+        'plugins': searx.plugins.plugins,
+        'version': {
+            'node': os.getenv('NODE_MINIMUM_VERSION')
+        },
     },
 }
 

docs/dev/makefile.rst

@@ -29,7 +29,7 @@ Calling the ``help`` target gives a first overview (``make help``):
 
 .. _make install:
 
-Python Environment (``make install``)
+Python environment (``make install``)
 =====================================
 
 .. sidebar:: activate environment
@@ -112,6 +112,60 @@ from the YAML configuration:
 - ``SEARXNG_BIND_ADDRESS`` from :ref:`server.bind_address <settings global server>`
 - ``SEARXNG_PORT`` from :ref:`server.port <settings global server>`
 
+.. _make node.env:
+
+Node.js environment (``make node.env``)
+=======================================
+
+.. _Node.js: https://nodejs.org/
+.. _nvm: https://github.com/nvm-sh
+.. _npm: https://www.npmjs.com/
+
+.. jinja:: searx
+
+   Node.js_ version {{version.node}} or higher is required to build the themes.
+   If the requirement is not met, the build chain uses nvm_ (Node Version
+   Manager) to install latest LTS of Node.js_ locally: there is no need to
+   install nvm_ or npm_ on your system.
+
+Use ``make nvm.status`` to get the current status of you Node.js_ and nvm_ setup.
+
+Here is the output you will typically get on a Ubuntu 20.04 system which serves
+only a `no longer active <https://nodejs.org/en/about/releases/>`_ Release
+`Node.js v10.19.0 <https://packages.ubuntu.com/focal/nodejs>`_.
+
+::
+
+  $ make nvm.status
+  INFO:  Node.js is installed at /usr/bin/node
+  INFO:  Node.js is version v10.19.0
+  WARN:  minimal Node.js version is 16.13.0
+  INFO:  npm is installed at /usr/bin/npm
+  INFO:  npm is version 6.14.4
+  WARN:  NVM is not installed
+  INFO:  to install NVM and Node.js (LTS) use: manage nvm install --lts
+
+To install you can also use :ref:`make nvm.nodejs`
+
+.. _make nvm.nodejs:
+
+``make nvm.nodejs``
+===================
+
+Install latest Node.js_ LTS locally (uses nvm_)::
+
+  $ make nvm.nodejs
+  INFO:  install (update) NVM at /share/searxng/.nvm
+  INFO:  clone: https://github.com/nvm-sh/nvm.git
+  ...
+  Downloading and installing node v16.13.0...
+  ...
+  INFO:  Node.js is installed at searxng/.nvm/versions/node/v16.13.0/bin/node
+  INFO:  Node.js is version v16.13.0
+  INFO:  npm is installed at searxng/.nvm/versions/node/v16.13.0/bin/npm
+  INFO:  npm is version 8.1.0
+  INFO:  NVM is installed at searxng/.nvm
+
 .. _make run:
 
 ``make run``
@@ -133,14 +187,16 @@ browser (:man:`xdg-open`)::
 ``make clean``
 ==============
 
-Drop all intermediate files, all builds, but keep sources untouched.  Before
-calling ``make clean`` stop all processes using :ref:`make install`. ::
+Drops all intermediate files, all builds, but keep sources untouched.  Before
+calling ``make clean`` stop all processes using the :ref:`make install` or
+:ref:`make node.env`. ::
 
    $ make clean
    CLEAN     pyenv
-   PYENV     [virtualenv] drop ./local/py3
-   CLEAN     docs -- ./build/docs ./dist/docs
-   CLEAN     locally installed npm dependencies
+   PYENV     [virtualenv] drop local/py3
+   CLEAN     docs -- build/docs dist/docs
+   CLEAN     themes -- locally installed npm dependencies
+   ...
    CLEAN     test stuff
    CLEAN     common files
 

docs/dev/quickstart.rst

@@ -5,6 +5,7 @@ Development Quickstart
 ======================
 
 .. _npm: https://www.npmjs.com/
+.. _Node.js: https://nodejs.org/
 
 SearXNG loves developers, just clone and start hacking.  All the rest is done for
 you simply by using :ref:`make <makefile>`.
@@ -24,37 +25,27 @@ choose a meaningful commit message and we are happy to receive your pull
 request. To not end in *wild west* we have some directives, please pay attention
 to our ":ref:`how to contribute`" guideline.
 
-If you implement themes, you will need to compile styles and JavaScript before
-*run*.
+If you implement themes, you will need to setup a :ref:`make node.env` once:
+
+.. code:: sh
+
+   make node.env
+
+Before you call *make run* (2.), you need to compile the modified styles and
+JavaScript:
 
 .. code:: sh
 
    make themes.all
 
-Don't forget to install npm_ first.
-
-.. tabs::
-
-   .. group-tab:: Ubuntu / debian
-
+Alternatively you can also compile selective the theme you have modified,
+e.g. the *simple* theme.
 .. code:: sh
 
-         sudo -H apt-get install npm
-
-   .. group-tab:: Arch Linux
-
-      .. code-block:: sh
-
-         sudo -H pacman -S npm
-
-   .. group-tab::  Fedora / RHEL
-
-      .. code-block:: sh
-
-	 sudo -H dnf install npm
+   make themes.simple
 
 If you finished your *tests* you can start to commit your changes.  To separate
-the changed code from the build products first run:
+the modified source code from the build products first run:
 
 .. code:: sh
 

manage

@@ -1,11 +1,16 @@
 #!/usr/bin/env bash
 # -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-
 # SPDX-License-Identifier: AGPL-3.0-or-later
-# shellcheck disable=SC2031
+
+# shellcheck disable=SC2034
+main_cmd="$(basename "$0")"
 
 # shellcheck source=utils/lib.sh
 source "$(dirname "${BASH_SOURCE[0]}")/utils/lib.sh"
 
+# shellcheck source=utils/lib.sh
+source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_nvm.sh"
+
 # shellcheck source=utils/lib_static.sh
 source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_static.sh"
 
@@ -14,6 +19,7 @@ source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_static.sh"
 PYOBJECTS="searx"
 PY_SETUP_EXTRAS='[test]'
 GECKODRIVER_VERSION="v0.28.0"
+export NODE_MINIMUM_VERSION="16.13.0"
 # SPHINXOPTS=
 
 pylint.FILES() {
@@ -41,6 +47,7 @@ PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES="supported_languages,language_aliases,log
 PYLINT_OPTIONS="-m pylint -j 0 --rcfile .pylintrc"
 
 help() {
+    nvm.help
     cat <<EOF
 buildenv:
   rebuild ./utils/brand.env
@@ -63,9 +70,12 @@ docker.:
 gecko.driver:
   download & install geckodriver if not already installed (required for
   robot_tests)
+EOF
+    nvm.help
+    cat <<EOF
 node.:
   env       : download & install npm dependencies locally
-  clean     : drop npm installations
+  clean     : drop locally npm installations
 py.:
   build     : Build python packages at ./${PYDIST}
   clean     : delete virtualenv and intermediate py files
@@ -499,10 +509,9 @@ gecko.driver() {
 }
 
 node.env() {
-    if ! required_commands npm; then
-        info_msg "to install build tools use::"
-        info_msg "   sudo -H ./utils/searx.sh install buildhost"
-        die 1 "install needed build tools first"
+    if ! nvm.min_node "${NODE_MINIMUM_VERSION}"; then
+        info_msg "install Node.js by NVM"
+        nvm.nodejs
     fi
 
     (   set -e
@@ -521,7 +530,7 @@ node.clean() {
         build_msg CLEAN "npm is not installed / ignore npm dependencies"
         return 0
     fi
-    build_msg CLEAN "locally installed npm dependencies"
+    build_msg CLEAN "themes -- locally installed npm dependencies"
     (   set -e
         npm --prefix searx/static/themes/oscar run clean
         npm --prefix searx/static/themes/simple run clean

utils/lib_nvm.sh

@@ -0,0 +1,171 @@
+#!/usr/bin/env bash
+# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-
+# SPDX-License-Identifier: AGPL-3.0-or-later
+#
+# Tools to install and maintain NVM versions manager for Node.js
+#
+# [1] https://github.com/nvm-sh/nvm
+
+# https://github.com/koalaman/shellcheck/issues/356#issuecomment-853515285
+# shellcheck source=utils/lib.sh
+. /dev/null
+
+declare main_cmd
+
+# configure nvm environment
+# -------------------------
+
+NVM_LOCAL_FOLDER=.nvm
+
+[[ -z "${NVM_GIT_URL}" ]] &&  NVM_GIT_URL="https://github.com/nvm-sh/nvm.git"
+[[ -z "${NVM_MIN_NODE_VER}" ]] && NVM_MIN_NODE_VER="16.13.0"
+
+# initalize nvm environment
+# -------------------------
+
+nvm.env() {
+    source "${NVM_DIR}/nvm.sh"
+    source "${NVM_DIR}/bash_completion"
+}
+
+nvm.is_installed() {
+    # is true if NVM is installed / in $HOME or even in <repo-root>/.nvm
+    [[ -d "${NVM_DIR}" ]]
+}
+
+if [[ -z "${NVM_DIR}" ]]; then
+    # nvm is not pre-intalled in $HOME.  Prepare for using nvm from <repo-root>
+    NVM_DIR="$(git rev-parse --show-toplevel)/${NVM_LOCAL_FOLDER}"
+fi
+export NVM_DIR
+
+if nvm.is_installed; then
+    [ "$VERBOSE" = "1" ] && info_msg "source NVM environment from ${NVM_DIR}"
+    nvm.env
+else
+    # if nvm is not installed, use this function as a wrapper
+    nvm() {
+        nvm.ensure
+        nvm "$@"
+    }
+fi
+
+# implement nvm functions
+# -----------------------
+
+nvm.is_local() {
+    # is true if NVM is installed in <repo-root>/.nvm
+    [ "${NVM_DIR}" = "$(git rev-parse --show-toplevel)/${NVM_LOCAL_FOLDER}" ]
+}
+
+nvm.min_node(){
+
+    # usage:  nvm.min_node 16.3.0
+    #
+    # Is true if minimal Node.js version is installed.
+
+    local min_v
+    local node_v
+    local higher_v
+
+    if ! command -v node >/dev/null; then
+        wanr_msg "Node.js is not yet installed"
+        return 42
+    fi
+
+    min_v="${1}"
+    node_v="$(node --version)"
+    node_v="${node_v:1}" # remove 'v' from 'v16.3.0'
+    if ! [ "${min_v}" = "${node_v}" ]; then
+        higher_v="$(echo -e "$min_v\n${node_v}" | sort -Vr | head -1)"
+        if [ "${min_v}" = "${higher_v}" ]; then
+            return 42
+        fi
+    fi
+}
+
+# implement nvm command line
+# --------------------------
+
+nvm.help(){
+    cat <<EOF
+nvm.: use nvm (without dot) to execute nvm commands directly
+  install   : install NVM locally at $(git rev-parse --show-toplevel)/${NVM_LOCAL_FOLDER}
+  clean     : remove NVM installation
+  status    : prompt some status informations about nvm & node
+  nodejs    : install Node.js latest LTS
+  bash      : start bash interpreter with NVM environment sourced
+EOF
+}
+
+nvm.install() {
+    local NVM_VERSION_TAG
+    info_msg "install (update) NVM at ${NVM_DIR}"
+    if [[ -d "${NVM_DIR}" ]] ; then
+        info_msg "already cloned at: ${NVM_DIR}"
+        pushd "${NVM_DIR}" &> /dev/null
+        git fetch --all | prefix_stdout "  ${_Yellow}||${_creset} "
+    else
+        info_msg "clone: ${NVM_GIT_URL}"
+        git clone "${NVM_GIT_URL}" "${NVM_DIR}" 2>&1 | prefix_stdout "  ${_Yellow}||${_creset} "
+        pushd "${NVM_DIR}" &> /dev/null
+        git config --local advice.detachedHead false
+    fi
+    NVM_VERSION_TAG="$(git rev-list --tags --max-count=1)"
+    NVM_VERSION_TAG="$(git describe --abbrev=0 --tags --match "v[0-9]*" "${NVM_VERSION_TAG}")"
+    info_msg "checkout ${NVM_VERSION_TAG}"
+    git checkout "${NVM_VERSION_TAG}" 2>&1 | prefix_stdout "  ${_Yellow}||${_creset} "
+    nvm.env
+}
+
+nvm.clean() {
+    if ! nvm.is_installed; then
+        info_msg "NVM is not installed"
+        return 42
+    fi
+    if ! nvm.is_local; then
+        info_msg "can't remove NVM from ${NVM_DIR}"
+        return 42
+    fi
+    rm -rf "${NVM_DIR}"
+}
+
+nvm.status(){
+    if command -v node >/dev/null; then
+        info_msg "Node.js is installed at $(command -v node)"
+        info_msg "Node.js is version $(node --version)"
+        if ! nvm.min_node "${NVM_MIN_NODE_VER}"; then
+            warn_msg "minimal Node.js version is ${NVM_MIN_NODE_VER}"
+        fi
+    else
+        warn_msg "Node.js is mot installed"
+    fi
+    if command -v npm >/dev/null; then
+        info_msg "npm is installed at $(command -v npm)"
+        info_msg "npm is version $(npm --version)"
+    else
+        warn_msg "npm is not installed"
+    fi
+    if nvm.is_installed; then
+        info_msg "NVM is installed at ${NVM_DIR}"
+    else
+        warn_msg "NVM is not installed"
+        info_msg "to install NVM and Node.js (LTS) use: ${main_cmd} nvm install --lts"
+    fi
+}
+
+nvm.nodejs(){
+    nvm install --lts
+    nvm.status
+}
+
+nvm.bash() {
+    nvm.ensure
+    bash --init-file <(cat "${NVM_DIR}/nvm.sh" "${NVM_DIR}/bash_completion")
+}
+
+nvm.ensure() {
+    if ! nvm.is_installed; then
+        nvm.install
+    fi
+}

utils/searx.sh

@@ -43,8 +43,7 @@ shellcheck"
 BUILD_PACKAGES_debian="\
 firefox graphviz imagemagick texlive-xetex librsvg2-bin
 texlive-latex-recommended texlive-extra-utils fonts-dejavu
-latexmk
-npm"
+latexmk"
 
 # pacman packages
 SEARX_PACKAGES_arch="\
@@ -55,8 +54,7 @@ shellcheck"
 
 BUILD_PACKAGES_arch="\
 firefox graphviz imagemagick texlive-bin extra/librsvg
-texlive-core texlive-latexextra ttf-dejavu
-npm"
+texlive-core texlive-latexextra ttf-dejavu"
 
 # dnf packages
 SEARX_PACKAGES_fedora="\
@@ -69,8 +67,7 @@ BUILD_PACKAGES_fedora="\
 firefox graphviz graphviz-gd ImageMagick librsvg2-tools
 texlive-xetex-bin texlive-collection-fontsrecommended
 texlive-collection-latex dejavu-sans-fonts dejavu-serif-fonts
-dejavu-sans-mono-fonts
-npm"
+dejavu-sans-mono-fonts"
 
 # yum packages
 #