[enh] implement all build task in a bash script (manage)

note: in further patches script 'manage' will replace 'manage.sh'

pyenv.* :
  assert    : build virtualenv if not exists
  intsall   : developer install of searx into virtualenv
  unintsall : uninstall developer installation
  cmd ...   : run command ... in virtualenv
  OK        : test if virtualenv is OK
pypi.upload:
  Upload python packages to PyPi (to test use pypi.upload.test)
pybuild :
  Build python packages at ./${PYDIST}
pyclean :
  delete virtualenv and intermediate py files
test.* :
  pylint    : lint PYLINT_FILES, searx/engines, searx & tests
  pep8      : pycodestyle (pep8) for all files except PYLINT_FILES
  unit      : run unit tests
  coverage  : run unit tests with coverage
  robot     : run robot test
  clean     : clean intermediate test stuff
node.* :
  env       : download & install npm dependencies locally
  clean     : drop npm installations
buildenv :
  rebuild ./utils/brand.env
data.* :
  all       : update searx/languages.py and ./data/*
  languages : update searx/data/engines_languages.json & searx/languages.py
  useragents: update searx/data/useragents.json with the most recent versions of Firefox.
themes.* :
  all       : build all themes
  oscar     : build oscar theme
  simple    : build simple theme
  bootstrap : less compile bootstrap.min.css CSS
babel.compile :
  pybabel compile ./searx/translations
docs.* :
  html      : build HTML documentation
  gh-pages  : deploy on gh-pages branch
  autobuild : autobuild HTML documentation while editing
  prebuild  : build reST include files (./${DOCS_BUILD}/includes)
  clean     : clean documentation build
docker.build [push] :
  build (and push) docker image
gecko.driver :
  download & install geckodriver if not already installed (required for
  robot_tests)

Signed-off-by: Markus Heiser <markus@darmarit.de>

manage

@@ -0,0 +1,506 @@
+#!/usr/bin/env bash
+# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-
+# SPDX-License-Identifier: AGPL-3.0-or-later
+# shellcheck disable=SC2031
+
+# shellcheck source=utils/lib.sh
+source "$(dirname "${BASH_SOURCE[0]}")/utils/lib.sh"
+# shellcheck source=utils/brand.env
+source "${REPO_ROOT}/utils/brand.env"
+source_dot_config
+
+# config
+
+PY_SETUP_EXTRAS='[test]'
+NPM_PACKAGES="less@2.7 less-plugin-clean-css grunt-cli"
+GECKODRIVER_VERSION="v0.28.0"
+# SPHINXOPTS=
+
+# These py files are linted by test.pylint(), all other files are linted by
+# test.pep8()
+PYLINT_FILES=(
+    searx/preferences.py
+    searx/testing.py
+    searx/engines/gigablast.py
+    searx/engines/deviantart.py
+    searx/engines/digg.py
+    searx/engines/google.py
+    searx/engines/google_news.py
+    searx/engines/google_videos.py
+    searx/engines/google_images.py
+    searx/engines/mediathekviewweb.py
+)
+
+PYLINT_SEARX_DISABLE_OPTION="\
+I,C,R,\
+W0105,W0212,W0511,W0603,W0613,W0621,W0702,W0703,W1401,\
+E1136"
+PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES="supported_languages,language_aliases"
+PYLINT_OPTIONS="-m pylint -j 0 --rcfile .pylintrc"
+
+help() {
+    cat <<EOF
+buildenv
+  rebuild ./utils/brand.env
+babel.compile
+  pybabel compile ./searx/translations
+data.*
+  all       : update searx/languages.py and ./data/*
+  languages : update searx/data/engines_languages.json & searx/languages.py
+  useragents: update searx/data/useragents.json with the most recent versions of Firefox.
+docs.*
+  html      : build HTML documentation
+  live      : autobuild HTML documentation while editing
+  gh-pages  : deploy on gh-pages branch
+  prebuild  : build reST include files (./${DOCS_BUILD}/includes)
+  clean     : clean documentation build
+docker
+  build     : build docker image
+  push      : build and push docker image
+gecko.driver
+  download & install geckodriver if not already installed (required for
+  robot_tests)
+node.*
+  env       : download & install npm dependencies locally
+  clean     : drop npm installations
+py.*
+  build     : Build python packages at ./${PYDIST}
+  clean     : delete virtualenv and intermediate py files
+pyenv.* :
+  install   : developer install of searx into virtualenv
+  uninstall : uninstall developer installation
+  cmd ...   : run command ... in virtualenv
+  OK        : test if virtualenv is OK
+pypi.upload:
+  Upload python packages to PyPi (to test use pypi.upload.test)
+test.* :
+  pylint    : lint PYLINT_FILES, searx/engines, searx & tests
+  pep8      : pycodestyle (pep8) for all files except PYLINT_FILES
+  unit      : run unit tests
+  coverage  : run unit tests with coverage
+  robot     : run robot test
+  clean     : clean intermediate test stuff
+themes.* :
+  all       : build all themes
+  oscar     : build oscar theme
+  simple    : build simple theme
+  bootstrap : less compile bootstrap.min.css CSS
+EOF
+}
+
+
+if [ "$VERBOSE" = "1" ]; then
+    SPHINX_VERBOSE="-v"
+    PYLINT_VERBOSE="-v"
+fi
+
+# needed by sphinx-docs
+export DOCS_BUILD
+
+buildenv() {
+    SEARX_DEBUG=1 pyenv.cmd python utils/build_env.py 2>&1 \
+        | prefix_stdout "${_Blue}BUILDENV${_creset}  "
+    return "${PIPESTATUS[0]}"
+}
+
+babel.compile() {
+    build_msg BABEL compile
+    pyenv.cmd pybabel compile -d "${REPO_ROOT}/searx/translations"
+    dump_return $?
+}
+
+
+data.all() {
+    data.languages
+    data.useragents
+    build_msg DATA "update searx/data/ahmia_blacklist.txt"
+    pyenv.cmd python utils/fetch_ahmia_blacklist.py
+    build_msg DATA "update searx/data/wikidata_units.json"
+    pyenv.cmd python utils/fetch_wikidata_units.py
+    build_msg DATA "update searx/data/currencies.json"
+    pyenv.cmd python utils/fetch_currencies.py
+}
+
+
+data.languages() {
+    (   set -e
+        build_msg ENGINES "fetch languages .."
+        pyenv.cmd python utils/fetch_languages.py
+        build_msg ENGINES "update update searx/languages.py"
+        build_msg DATA "update searx/data/engines_languages.json"
+    )
+    dump_return $?
+}
+
+data.useragents() {
+    build_msg DATA "update searx/data/useragents.json"
+    pyenv.cmd python utils/fetch_firefox_version.py
+    dump_return $?
+}
+
+docs.prebuild() {
+    build_msg DOCS "build ${DOCS_BUILD}/includes"
+    (
+        set -e
+        [ "$VERBOSE" = "1" ] && set -x
+        mkdir -p "${DOCS_BUILD}/includes"
+        ./utils/searx.sh doc   | cat > "${DOCS_BUILD}/includes/searx.rst"
+        ./utils/filtron.sh doc | cat > "${DOCS_BUILD}/includes/filtron.rst"
+        ./utils/morty.sh doc   | cat > "${DOCS_BUILD}/includes/morty.rst"
+    )
+    dump_return $?
+}
+
+docker.push() {
+    docker.build push
+}
+
+# shellcheck disable=SC2119
+docker.build() {
+    pyenv.install
+
+    build_msg DOCKER build
+    # run installation in a subprocess and activate pyenv
+
+    # See https://www.shellcheck.net/wiki/SC1001 and others ..
+    # shellcheck disable=SC2031,SC2230,SC2002,SC2236,SC2143,SC1001
+    (   set -e
+        # shellcheck source=/dev/null
+        source "${PY_ENV_BIN}/activate"
+
+        # Check if it is a git repository
+        if [ ! -d .git ]; then
+	    die 1 "This is not Git repository"
+        fi
+        if [ ! -x "$(which git)" ]; then
+	    die 1 "git is not installed"
+        fi
+
+        if ! git remote get-url origin 2> /dev/null; then
+	    die 1 "there is no remote origin"
+        fi
+
+        # This is a git repository
+
+        # "git describe" to get the Docker version (for example : v0.15.0-89-g0585788e)
+        # awk to remove the "v" and the "g"
+        SEARX_GIT_VERSION=$(git describe --match "v[0-9]*\.[0-9]*\.[0-9]*" HEAD 2>/dev/null | awk -F'-' '{OFS="-"; $1=substr($1, 2); if ($3) { $3=substr($3, 2); }  print}')
+
+        # add the suffix "-dirty" if the repository has uncommited change
+        # /!\ HACK for searx/searx: ignore utils/brand.env
+        git update-index -q --refresh
+        if [ ! -z "$(git diff-index --name-only HEAD -- | grep -v 'utils/brand.env')" ]; then
+	    SEARX_GIT_VERSION="${SEARX_GIT_VERSION}-dirty"
+        fi
+
+        # Get the last git commit id, will be added to the Searx version (see Dockerfile)
+        VERSION_GITCOMMIT=$(echo "$SEARX_GIT_VERSION" | cut -d- -f2-4)
+        build_msg DOCKER "Last commit : $VERSION_GITCOMMIT"
+
+        # Check consistency between the git tag and the searx/version.py file
+        # /! HACK : parse Python file with bash /!
+        # otherwise it is not possible build the docker image without all Python
+        # dependencies ( version.py loads __init__.py )
+        # SEARX_PYTHON_VERSION=$(python3 -c "import six; import searx.version; six.print_(searx.version.VERSION_STRING)")
+        SEARX_PYTHON_VERSION=$(cat searx/version.py | grep "\(VERSION_MAJOR\|VERSION_MINOR\|VERSION_BUILD\) =" | cut -d\= -f2 | sed -e 's/^[[:space:]]*//' | paste -sd "." -)
+        if [ "$(echo "$SEARX_GIT_VERSION" | cut -d- -f1)" != "$SEARX_PYTHON_VERSION" ]; then
+	    err_msg "git tag:          $SEARX_GIT_VERSION"
+	    err_msg "searx/version.py: $SEARX_PYTHON_VERSION"
+	    die 1 "Inconsistency between the last git tag and the searx/version.py file"
+        fi
+
+        # define the docker image name
+        GITHUB_USER=$(echo "${GIT_URL}" | sed 's/.*github\.com\/\([^\/]*\).*/\1/')
+        SEARX_IMAGE_NAME="${SEARX_IMAGE_NAME:-${GITHUB_USER:-searx}/searx}"
+
+        # build Docker image
+        build_msg DOCKER "Building image ${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}"
+        sudo docker build \
+         --build-arg GIT_URL="${GIT_URL}" \
+         --build-arg SEARX_GIT_VERSION="${SEARX_GIT_VERSION}" \
+         --build-arg VERSION_GITCOMMIT="${VERSION_GITCOMMIT}" \
+         --build-arg LABEL_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \
+         --build-arg LABEL_VCS_REF="$(git rev-parse HEAD)" \
+         --build-arg LABEL_VCS_URL="${GIT_URL}" \
+	 --build-arg TIMESTAMP_SETTINGS="$(git log -1 --format="%cd" --date=unix -- searx/settings.yml)" \
+	 --build-arg TIMESTAMP_UWSGI="$(git log -1 --format="%cd" --date=unix -- dockerfiles/uwsgi.ini)" \
+         -t "${SEARX_IMAGE_NAME}:latest" -t "${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}" .
+
+        if [ "$1" = "push" ]; then
+	    sudo docker push "${SEARX_IMAGE_NAME}:latest"
+	    sudo docker push "${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}"
+        fi
+    )
+    dump_return $?
+}
+
+# shellcheck disable=SC2119
+gecko.driver() {
+    pyenv.install
+
+    build_msg INSTALL "gecko.driver"
+    # run installation in a subprocess and activate pyenv
+    (   set -e
+        # shellcheck source=/dev/null
+        source "${PY_ENV_BIN}/activate"
+
+        # TODO : check the current geckodriver version
+        geckodriver -V > /dev/null 2>&1 || NOTFOUND=1
+        set +e
+        if [ -z "$NOTFOUND" ]; then
+            build_msg INSTALL "geckodriver already installed"
+            return
+        fi
+        PLATFORM="$(python3 -c 'import platform; print(platform.system().lower(), platform.architecture()[0])')"
+        case "$PLATFORM" in
+            "linux 32bit" | "linux2 32bit") ARCH="linux32";;
+            "linux 64bit" | "linux2 64bit") ARCH="linux64";;
+            "windows 32 bit") ARCH="win32";;
+            "windows 64 bit") ARCH="win64";;
+            "mac 64bit") ARCH="macos";;
+        esac
+        GECKODRIVER_URL="https://github.com/mozilla/geckodriver/releases/download/$GECKODRIVER_VERSION/geckodriver-$GECKODRIVER_VERSION-$ARCH.tar.gz";
+
+        build_msg GECKO "Installing ${PY_ENV_BIN}/geckodriver from $GECKODRIVER_URL"
+
+        FILE="$(mktemp)"
+        wget -qO "$FILE" -- "$GECKODRIVER_URL" && tar xz -C "${PY_ENV_BIN}" -f "$FILE" geckodriver
+        rm -- "$FILE"
+        chmod 755 -- "${PY_ENV_BIN}/geckodriver"
+    )
+    dump_return $?
+}
+
+node.env() {
+    local err=0
+    pushd "${REPO_ROOT}" &> /dev/null
+    # shellcheck disable=SC2230
+    which npm &> /dev/null || die 1 'node.env - npm is not found!'
+
+    (   set -e
+        # shellcheck disable=SC2030
+        PATH="$(npm bin):$PATH"
+        export PATH
+
+        build_msg INSTALL "npm install $NPM_PACKAGES"
+        # shellcheck disable=SC2086
+        npm install $NPM_PACKAGES
+
+        cd "${REPO_ROOT}/searx/static/themes/oscar"
+        build_msg INSTALL "($(pwd)) npm install"
+        npm install
+
+        build_msg INSTALL "($(pwd)) npm install"
+        cd "${REPO_ROOT}/searx/static/themes/simple"
+        npm install
+    )
+    err=$?
+    popd &> /dev/null
+    dump_return "$err"
+}
+
+node.clean() {
+
+    build_msg CLEAN "locally installed npm dependencies"
+    rm -rf \
+       ./node_modules  \
+       ./package-lock.json \
+       ./searx/static/themes/oscar/package-lock.json \
+       ./searx/static/themes/oscar/node_modules \
+       ./searx/static/themes/simple/package-lock.json \
+       ./searx/static/themes/simple/node_modules
+    dump_return $?
+}
+
+py.build() {
+    build_msg BUILD "[pylint] python package ${PYDIST}"
+    pyenv.cmd python setup.py \
+              sdist -d "${PYDIST}" \
+              bdist_wheel --bdist-dir "${PYBUILD}" -d "${PYDIST}"
+}
+
+py.clean() {
+    build_msg CLEAN pyenv
+    (   set -e
+        pyenv.drop
+        [ "$VERBOSE" = "1" ] && set -x
+        rm -rf "${PYDIST}" "${PYBUILD}" "${PY_ENV}" ./.tox ./*.egg-info
+        find . -name '*.pyc' -exec rm -f {} +
+        find . -name '*.pyo' -exec rm -f {} +
+        find . -name __pycache__ -exec rm -rf {} +
+    )
+}
+
+pyenv.check() {
+    cat  <<EOF
+import yaml
+print('import yaml --> OK')
+EOF
+}
+
+pyenv.install() {
+
+    if ! pyenv.OK; then
+        py.clean > /dev/null
+    fi
+    if pyenv.install.OK > /dev/null; then
+        return 0
+    fi
+    (   set -e
+        py.clean > /dev/null
+        pyenv
+    	build_msg PYENV "[install] pip install -e 'searx${PY_SETUP_EXTRAS}'"
+    	"${PY_ENV_BIN}/python" -m pip install -e ".${PY_SETUP_EXTRAS}"
+        buildenv
+    ) || die 42 "error while build & install pyenv (${PY_ENV_BIN})"
+}
+
+pyenv.uninstall() {
+    build_msg PYENV "[pyenv.uninstall] uninstall packages: ${PYOBJECTS}"
+    pyenv.cmd python setup.py develop --uninstall 2>&1 \
+        | prefix_stdout "${_Blue}PYENV     ${_creset}[pyenv.uninstall] "
+
+}
+
+pypi.upload() {
+    py.clean
+    py.build
+    # https://github.com/pypa/twine
+    pyenv.cmd twine upload "${PYDIST}"/*
+}
+
+pypi.upload.test() {
+    py.clean
+    py.build
+    pyenv.cmd twine upload -r testpypi "${PYDIST}"/*
+}
+
+test.pylint() {
+    # shellcheck disable=SC2086
+    (   set -e
+        build_msg TEST "[pylint] \$PYLINT_FILES"
+        pyenv.cmd python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \
+            "${PYLINT_FILES[@]}"
+
+        build_msg TEST "[pylint] searx/engines"
+        pyenv.cmd python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \
+            --disable="${PYLINT_SEARX_DISABLE_OPTION}" \
+            --additional-builtins="${PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES}" \
+            searx/engines
+
+        build_msg TEST "[pylint] searx tests"
+        pyenv.cmd python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \
+            --disable="${PYLINT_SEARX_DISABLE_OPTION}" \
+	    --ignore=searx/engines \
+	    searx tests
+    )
+    dump_return $?
+}
+
+test.pep8() {
+    build_msg TEST 'pycodestyle (formerly pep8)'
+    local _exclude=""
+    printf -v _exclude '%s, ' "${PYLINT_FILES[@]}"
+    pyenv.cmd pycodestyle \
+              --exclude="searx/static, searx/languages.py, $_exclude " \
+              --max-line-length=120 \
+              --ignore "E117,E252,E402,E722,E741,W503,W504,W605" \
+              searx tests
+    dump_return $?
+}
+
+test.unit() {
+    build_msg TEST 'tests/unit'
+    pyenv.cmd python -m nose2 -s tests/unit
+    dump_return $?
+}
+
+test.coverage() {
+    build_msg TEST 'unit test coverage'
+    (   set -e
+        pyenv.cmd python -m nose2 -C --log-capture --with-coverage --coverage searx -s tests/unit
+        pyenv.cmd coverage report
+        pyenv.cmd coverage html
+    )
+    dump_return $?
+}
+
+test.robot() {
+    build_msg TEST 'robot'
+    gecko.driver
+    PYTHONPATH=. pyenv.cmd python searx/testing.py robot
+    dump_return $?
+}
+
+test.clean() {
+    build_msg CLEAN  "test stuff"
+    rm -rf geckodriver.log .coverage coverage/
+    dump_return $?
+}
+
+themes.all() {
+    (   set -e
+        node.env
+        themes.oscar
+        themes.simple
+        themes.bootstrap
+    )
+    dump_return $?
+}
+
+themes.oscar() {
+    local gruntfile=searx/static/themes/oscar/gruntfile.js
+    build_msg GRUNT "${gruntfile}"
+    PATH="$(npm bin):$PATH" grunt --gruntfile "${gruntfile}"
+    dump_return $?
+}
+
+themes.simple() {
+    local gruntfile=searx/static/themes/simple/gruntfile.js
+    build_msg GRUNT "${gruntfile}"
+    PATH="$(npm bin):$PATH" grunt --gruntfile "${gruntfile}"
+    dump_return $?
+}
+
+themes.bootstrap() {
+    build_msg LESSC "searx/static/css/bootstrap.min.css"
+    PATH="$(npm bin):$PATH" lessc \
+        --clean-css="--s1 --advanced --compatibility=ie9" \
+        "searx/static/less/bootstrap/bootstrap.less" \
+        "searx/static/css/bootstrap.min.css"
+    dump_return $?
+}
+
+# shellcheck disable=SC2119
+main() {
+
+    local _type
+    local cmd="$1"; shift
+
+    if [ "$cmd" == "" ]; then
+        help
+        err_msg "missing command"
+        return 42
+    fi
+
+    case "$cmd" in
+        --getenv) var="$1"; echo "${!var}";;
+        --help) help;;
+        --*)
+            help
+            err_msg "unknown option $cmd"
+            return 42
+            ;;
+        *)
+            _type="$(type -t "$cmd")"
+            if [ "$_type" != 'function' ]; then
+                err_msg "unknown command $1 / use --help"
+                return 42
+            else
+                "$cmd" "$@"
+            fi
+            ;;
+    esac
+}
+
+main "$@"