#!/usr/bin/env bash # -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*- # SPDX-License-Identifier: AGPL-3.0-or-later # shellcheck disable=SC2034 main_cmd="$(basename "$0")" # shellcheck source=utils/lib.sh source "$(dirname "${BASH_SOURCE[0]}")/utils/lib.sh" # shellcheck source=utils/lib.sh source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_nvm.sh" # shellcheck source=utils/lib_static.sh source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_static.sh" # shellcheck source=utils/lib_go.sh source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_go.sh" # shellcheck source=utils/lib_redis.sh source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_redis.sh" PATH="${REPO_ROOT}/node_modules/.bin:${PATH}" # config PYOBJECTS="searx" PY_SETUP_EXTRAS='[test]' GECKODRIVER_VERSION="v0.30.0" export NODE_MINIMUM_VERSION="16.13.0" # SPHINXOPTS= BLACK_OPTIONS=("--target-version" "py37" "--line-length" "120" "--skip-string-normalization") BLACK_TARGETS=("--exclude" "searx/static,searx/languages.py" "--include" 'searxng.msg|\.pyi?$' "searx" "searxng_extra" "tests") pylint.FILES() { # List files tagged by comment: # # # lint: pylint # # These py files are linted by test.pylint() grep -l -r --include \*.py '^#[[:blank:]]*lint:[[:blank:]]*pylint' searx searxng_extra tests find . -name searxng.msg } YAMLLINT_FILES=() while IFS= read -r line; do YAMLLINT_FILES+=("$line") done <<< "$(git ls-files './tests/*.yml' './searx/*.yml' './utils/templates/etc/searxng/*.yml')" RST_FILES=( 'README.rst' ) PYLINT_SEARXNG_DISABLE_OPTION="\ I,C,R,\ W0105,W0212,W0511,W0603,W0613,W0621,W0702,W0703,W1401,\ E1136" PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES="supported_languages,language_aliases,logger,categories" PYLINT_OPTIONS="-m pylint -j 0 --rcfile .pylintrc" help() { nvm.help cat <&1 \ | prefix_stdout "${_Blue}BUILDENV${_creset} " ) return "${PIPESTATUS[0]}" } TRANSLATIONS_WORKTREE="$CACHE/translations" weblate.translations.worktree() { # Create git worktree ${TRANSLATIONS_WORKTREE} and checkout branch # 'translations' from Weblate's counterpart (weblate) of the SearXNG # (origin). # # remote weblate https://weblate.bubu1.eu/git/searxng/searxng/ ( set -e if ! git remote get-url weblate 2> /dev/null; then git remote add weblate https://weblate.bubu1.eu/git/searxng/searxng/ fi if [ -d "${TRANSLATIONS_WORKTREE}" ]; then pushd . cd "${TRANSLATIONS_WORKTREE}" git reset --hard HEAD git pull origin translations popd else mkdir -p "${TRANSLATIONS_WORKTREE}" git worktree add "${TRANSLATIONS_WORKTREE}" translations fi ) } weblate.to.translations() { # Update 'translations' branch of SearXNG (origin) with last additions from # Weblate. # 1. Check if Weblate is locked, if not die with error message # 2. On Weblate's counterpart (weblate), pull master and translations branch # from SearXNG (origin). # 3. Commit changes made in a Weblate object on Weblate's counterpart # (weblate). # 4. In translations worktree, merge changes of branch 'translations' from # remote 'weblate' and push it on branch 'translations' of 'origin' ( set -e pyenv.activate if [ "$(wlc lock-status)" != "locked: True" ]; then die 1 "weblate must be locked, currently: $(wlc lock-status)" fi # weblate: commit pending changes wlc pull wlc commit # get the translations in a worktree weblate.translations.worktree pushd "${TRANSLATIONS_WORKTREE}" git remote update weblate git merge weblate/translations git push popd ) dump_return $? } weblate.translations.commit() { # Update 'translations' branch of SearXNG (origin) with last additions from # Weblate. Copy the changes to the master branch, compile translations and # create a commit in the local branch (master) local existing_commit_hash commit_body commit_message exitcode ( set -e pyenv.activate # lock change on weblate wlc lock # get translations branch in git worktree (TRANSLATIONS_WORKTREE) weblate.translations.worktree existing_commit_hash=$(cd "${TRANSLATIONS_WORKTREE}"; git log -n1 --pretty=format:'%h') # pull weblate commits weblate.to.translations # copy the changes to the master branch cp -rv --preserve=mode,timestamps "${TRANSLATIONS_WORKTREE}/searx/translations" "searx" # compile translations build_msg BABEL 'compile translation catalogs into binary MO files' pybabel compile --statistics \ -d "searx/translations" # git add/commit (no push) commit_body=$(cd "${TRANSLATIONS_WORKTREE}"; git log --pretty=format:'%h - %as - %aN <%ae>' "${existing_commit_hash}..HEAD") commit_message=$(echo -e "[translations] update from Weblate\n\n${commit_body}") git add searx/translations git commit -m "${commit_message}" ) exitcode=$? ( # make sure to always unlock weblate set -e pyenv.cmd wlc unlock ) dump_return $exitcode } weblate.push.translations() { # Push *translation changes* from SearXNG (origin) to Weblate's counterpart # (weblate). # In branch master of SearXNG (origin) check for meaningful changes in # folder 'searx/translations', commit changes on branch 'translations' and # at least, pull updated branches on Weblate's counterpart (weblate). # 1. Create git worktree ${TRANSLATIONS_WORKTREE} and checkout branch # 'translations' from remote 'weblate'. # 2. Stop if there is no meaningful change in the 'master' branch (origin), # compared to the 'translations' branch (weblate), otherwise ... # 3. Update 'translations' branch of SearXNG (origin) with last additions # from Weblate. # 5. Notify Weblate to pull updated 'master' & 'translations' branch. local messages_pot diff_messages_pot last_commit_hash last_commit_detail \ exitcode messages_pot="${TRANSLATIONS_WORKTREE}/searx/translations/messages.pot" ( set -e pyenv.activate # get translations branch in git worktree (TRANSLATIONS_WORKTREE) weblate.translations.worktree # update messages.pot in the master branch build_msg BABEL 'extract messages from source files and generate POT file' pybabel extract -F babel.cfg \ -o "${messages_pot}" \ "searx/" # stop if there is no meaningful change in the master branch diff_messages_pot=$(cd "${TRANSLATIONS_WORKTREE}";\ git diff -- "searx/translations/messages.pot") if ! echo "$diff_messages_pot" | grep -qE "[\+\-](msgid|msgstr)"; then build_msg BABEL 'no changes detected, exiting' return 42 fi return 0 ) exitcode=$? if [ "$exitcode" -eq 42 ]; then return 0 fi if [ "$exitcode" -gt 0 ]; then return $exitcode fi ( set -e pyenv.activate # lock change on weblate # weblate may add commit(s) since the call to "weblate.translations.worktree". # this is not a problem because after this line, "weblate.to.translations" # calls again "weblate.translations.worktree" which calls "git pull" wlc lock # save messages.pot in the translations branch for later pushd "${TRANSLATIONS_WORKTREE}" git stash push popd # merge weblate commits into the translations branch weblate.to.translations # restore messages.pot in the translations branch pushd "${TRANSLATIONS_WORKTREE}" git stash pop popd # update messages.po files in the master branch build_msg BABEL 'update existing message catalogs from POT file' pybabel update -N \ -i "${messages_pot}" \ -d "${TRANSLATIONS_WORKTREE}/searx/translations" # git add/commit/push last_commit_hash=$(git log -n1 --pretty=format:'%h') last_commit_detail=$(git log -n1 --pretty=format:'%h - %as - %aN <%ae>' "${last_commit_hash}") pushd "${TRANSLATIONS_WORKTREE}" git add searx/translations git commit \ -m "[translations] update messages.pot and messages.po files" \ -m "From ${last_commit_detail}" git push popd # notify weblate to pull updated master & translations branch wlc pull ) exitcode=$? ( # make sure to always unlock weblate set -e pyenv.activate wlc unlock ) dump_return $exitcode } data.all() { ( set -e pyenv.activate data.languages data.useragents data.osm_keys_tags build_msg DATA "update searx/data/ahmia_blacklist.txt" python searxng_extra/update/update_ahmia_blacklist.py build_msg DATA "update searx/data/wikidata_units.json" python searxng_extra/update/update_wikidata_units.py build_msg DATA "update searx/data/currencies.json" python searxng_extra/update/update_currencies.py ) } data.languages() { ( set -e pyenv.activate build_msg ENGINES "fetch languages .." python searxng_extra/update/update_languages.py build_msg ENGINES "update update searx/languages.py" build_msg DATA "update searx/data/engines_languages.json" ) dump_return $? } data.useragents() { build_msg DATA "update searx/data/useragents.json" pyenv.cmd python searxng_extra/update/update_firefox_version.py dump_return $? } data.osm_keys_tags() { build_msg DATA "update searx/data/osm_keys_tags.json" pyenv.cmd python searxng_extra/update/update_osm_keys_tags.py dump_return $? } docs.prebuild() { build_msg DOCS "build ${DOCS_BUILD}/includes" ( set -e [ "$VERBOSE" = "1" ] && set -x mkdir -p "${DOCS_BUILD}/includes" ./utils/searxng.sh searxng.doc.rst > "${DOCS_BUILD}/includes/searxng.rst" pyenv.cmd searxng_extra/docs_prebuild ) dump_return $? } docker.push() { docker.build push } docker.buildx() { docker.build buildx } # shellcheck disable=SC2119 docker.build() { pyenv.install local SEARXNG_GIT_VERSION local VERSION_GITCOMMIT local GITHUB_USER local SEARXNG_IMAGE_NAME local BUILD build_msg DOCKER build # run installation in a subprocess and activate pyenv # See https://www.shellcheck.net/wiki/SC1001 and others .. # shellcheck disable=SC2031,SC2230,SC2002,SC2236,SC2143,SC1001 ( set -e pyenv.activate # Check if it is a git repository if [ ! -d .git ]; then die 1 "This is not Git repository" fi if [ ! -x "$(which git)" ]; then die 1 "git is not installed" fi if ! git remote get-url origin 2> /dev/null; then die 1 "there is no remote origin" fi # This is a git repository git update-index -q --refresh python -m searx.version freeze eval "$(python -m searx.version)" # Get the last git commit id VERSION_GITCOMMIT=$(echo "$VERSION_STRING" | cut -d- -f3) build_msg DOCKER "Last commit : $VERSION_GITCOMMIT" # define the docker image name GITHUB_USER=fauli1221/ponysearch SEARXNG_IMAGE_NAME="${SEARXNG_IMAGE_NAME:-${GITHUB_USER:-searxng}/searxng}" BUILD="build" if [ "$1" = "buildx" ]; then # buildx includes the push option CACHE_TAG="${SEARXNG_IMAGE_NAME}:latest-build-cache" BUILD="buildx build --platform linux/amd64,linux/arm64,linux/arm/v7 --push --cache-from=type=registry,ref=$CACHE_TAG --cache-to=type=registry,ref=$CACHE_TAG,mode=max" shift fi build_msg DOCKER "Build command: ${BUILD}" # build Docker image build_msg DOCKER "Building image ${SEARXNG_IMAGE_NAME}:${SEARXNG_GIT_VERSION}" # shellcheck disable=SC2086 docker $BUILD \ --build-arg BASE_IMAGE="${DEPENDENCIES_IMAGE_NAME}" \ --build-arg GIT_URL="${GIT_URL}" \ --build-arg SEARXNG_GIT_VERSION="${VERSION_STRING}" \ --build-arg VERSION_GITCOMMIT="${VERSION_GITCOMMIT}" \ --build-arg LABEL_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \ --build-arg LABEL_VCS_REF="$(git rev-parse HEAD)" \ --build-arg LABEL_VCS_URL="${GIT_URL}" \ --build-arg TIMESTAMP_SETTINGS="$(git log -1 --format="%cd" --date=unix -- searx/settings.yml)" \ --build-arg TIMESTAMP_UWSGI="$(git log -1 --format="%cd" --date=unix -- dockerfiles/uwsgi.ini)" \ -t "${SEARXNG_IMAGE_NAME}:latest" -t "${SEARXNG_IMAGE_NAME}:${VERSION_STRING}" . if [ "$1" = "push" ]; then docker push "${SEARXNG_IMAGE_NAME}:latest" docker push "${SEARXNG_IMAGE_NAME}:${SEARXNG_GIT_VERSION}" fi ) dump_return $? } # shellcheck disable=SC2119 gecko.driver() { pyenv.install build_msg INSTALL "gecko.driver" # run installation in a subprocess and activate pyenv ( set -e pyenv.activate INSTALLED_VERSION=$(geckodriver -V 2> /dev/null | head -1 | awk '{ print "v" $2}') || INSTALLED_VERSION="" set +e if [ "${INSTALLED_VERSION}" = "${GECKODRIVER_VERSION}" ]; then build_msg INSTALL "geckodriver already installed" return fi PLATFORM="$(python3 -c 'import platform; print(platform.system().lower(), platform.architecture()[0])')" case "$PLATFORM" in "linux 32bit" | "linux2 32bit") ARCH="linux32";; "linux 64bit" | "linux2 64bit") ARCH="linux64";; "windows 32 bit") ARCH="win32";; "windows 64 bit") ARCH="win64";; "mac 64bit") ARCH="macos";; esac GECKODRIVER_URL="https://github.com/mozilla/geckodriver/releases/download/$GECKODRIVER_VERSION/geckodriver-$GECKODRIVER_VERSION-$ARCH.tar.gz"; build_msg GECKO "Installing ${PY_ENV_BIN}/geckodriver from $GECKODRIVER_URL" FILE="$(mktemp)" wget -qO "$FILE" -- "$GECKODRIVER_URL" && tar xz -C "${PY_ENV_BIN}" -f "$FILE" geckodriver rm -- "$FILE" chmod 755 -- "${PY_ENV_BIN}/geckodriver" ) dump_return $? } nodejs.ensure() { if ! nvm.min_node "${NODE_MINIMUM_VERSION}"; then info_msg "install Node.js by NVM" nvm.nodejs fi } node.env() { nodejs.ensure ( set -e build_msg INSTALL "searx/static/themes/simple/package.json" npm --prefix searx/static/themes/simple install ) dump_return $? } node.env.devtools() { nodejs.ensure build_msg INSTALL "package.json: developer and CI tools" npm install } node.clean() { if ! required_commands npm 2>/dev/null; then build_msg CLEAN "npm is not installed / ignore npm dependencies" return 0 fi build_msg CLEAN "themes -- locally installed npm dependencies" ( set -e npm --prefix searx/static/themes/simple run clean ) dump_return $? } pygments.less() { build_msg PYGMENTS "searxng_extra/update/update_pygments.py" if ! pyenv.cmd python searxng_extra/update/update_pygments.py; then build_msg PYGMENTS "building LESS files for pygments failed" return 1 fi return 0 } py.build() { build_msg BUILD "python package ${PYDIST}" pyenv.cmd python setup.py \ sdist -d "${PYDIST}" \ bdist_wheel --bdist-dir "${PYBUILD}" -d "${PYDIST}" } py.clean() { build_msg CLEAN pyenv ( set -e pyenv.drop [ "$VERBOSE" = "1" ] && set -x rm -rf "${PYDIST}" "${PYBUILD}" "${PY_ENV}" ./.tox ./*.egg-info find . -name '*.pyc' -exec rm -f {} + find . -name '*.pyo' -exec rm -f {} + find . -name __pycache__ -exec rm -rf {} + ) } pyenv.check() { cat < OK') EOF } pyenv.install() { if ! pyenv.OK; then py.clean > /dev/null fi if pyenv.install.OK > /dev/null; then return 0 fi ( set -e pyenv build_msg PYENV "[install] pip install -e 'searx${PY_SETUP_EXTRAS}'" "${PY_ENV_BIN}/python" -m pip install -e ".${PY_SETUP_EXTRAS}" buildenv ) local exit_val=$? if [ ! $exit_val -eq 0 ]; then die 42 "error while pip install (${PY_ENV_BIN})" fi } pyenv.uninstall() { build_msg PYENV "[pyenv.uninstall] uninstall packages: ${PYOBJECTS}" pyenv.cmd python setup.py develop --uninstall 2>&1 \ | prefix_stdout "${_Blue}PYENV ${_creset}[pyenv.uninstall] " } pypi.upload() { py.clean py.build # https://github.com/pypa/twine pyenv.cmd twine upload "${PYDIST}"/* } pypi.upload.test() { py.clean py.build pyenv.cmd twine upload -r testpypi "${PYDIST}"/* } format.python() { build_msg TEST "[format.python] black \$BLACK_TARGETS" pyenv.cmd black "${BLACK_OPTIONS[@]}" "${BLACK_TARGETS[@]}" dump_return $? } test.yamllint() { build_msg TEST "[yamllint] \$YAMLLINT_FILES" pyenv.cmd yamllint --strict --format parsable "${YAMLLINT_FILES[@]}" dump_return $? } test.pylint() { # shellcheck disable=SC2086 ( set -e build_msg TEST "[pylint] \$PYLINT_FILES" pyenv.activate python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \ --additional-builtins="${PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES}" \ "${PYLINT_FILES[@]}" build_msg TEST "[pylint] searx/engines" python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \ --disable="${PYLINT_SEARXNG_DISABLE_OPTION}" \ --additional-builtins="${PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES}" \ searx/engines build_msg TEST "[pylint] searx tests" python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \ --disable="${PYLINT_SEARXNG_DISABLE_OPTION}" \ --ignore=searx/engines \ searx tests ) dump_return $? } test.pyright() { build_msg TEST "[pyright] static type check of python sources" node.env.devtools # We run Pyright in the virtual environment because Pyright # executes "python" to determine the Python version. build_msg TEST "[pyright] suppress warnings related to intentional monkey patching" pyenv.cmd npx --no-install pyright -p pyrightconfig-ci.json \ | grep -v ".py$" \ | grep -v '/engines/.*.py.* - warning: "logger" is not defined'\ | grep -v '/plugins/.*.py.* - error: "logger" is not defined'\ | grep -v '/engines/.*.py.* - warning: "supported_languages" is not defined' \ | grep -v '/engines/.*.py.* - warning: "language_aliases" is not defined' \ | grep -v '/engines/.*.py.* - warning: "categories" is not defined' dump_return $? } test.black() { build_msg TEST "[black] \$BLACK_TARGETS" pyenv.cmd black --check --diff "${BLACK_OPTIONS[@]}" "${BLACK_TARGETS[@]}" dump_return $? } test.unit() { build_msg TEST 'tests/unit' pyenv.cmd python -m nose2 -s tests/unit dump_return $? } test.coverage() { build_msg TEST 'unit test coverage' ( set -e pyenv.activate python -m nose2 -C --log-capture --with-coverage --coverage searx -s tests/unit coverage report coverage html ) dump_return $? } test.robot() { build_msg TEST 'robot' gecko.driver PYTHONPATH=. pyenv.cmd python -m tests.robot dump_return $? } test.rst() { build_msg TEST "[reST markup] ${RST_FILES[*]}" for rst in "${RST_FILES[@]}"; do pyenv.cmd rst2html.py --halt error "$rst" > /dev/null || die 42 "fix issue in $rst" done } test.pybabel() { TEST_BABEL_FOLDER="build/test/pybabel" build_msg TEST "[extract messages] pybabel" mkdir -p "${TEST_BABEL_FOLDER}" pyenv.cmd pybabel extract -F babel.cfg -o "${TEST_BABEL_FOLDER}/messages.pot" searx } test.clean() { build_msg CLEAN "test stuff" rm -rf geckodriver.log .coverage coverage/ dump_return $? } themes.all() { ( set -e pygments.less node.env themes.simple ) dump_return $? } themes.live() { local LIVE_THEME="${LIVE_THEME:-${1}}" case "${LIVE_THEME}" in simple) theme="searx/static/themes/${LIVE_THEME}" ;; '') die_caller 42 "missing theme argument" ;; *) die_caller 42 "unknown theme '${LIVE_THEME}' // [simple]'" ;; esac build_msg GRUNT "theme: $1 (live build)" nodejs.ensure cd "${theme}" { npm install npm run watch } 2>&1 \ | prefix_stdout "${_Blue}THEME ${1} ${_creset} " \ | grep -E --ignore-case --color 'error[s]?[:]? |warning[s]?[:]? |' } themes.simple() { ( set -e build_msg GRUNT "theme: simple" npm --prefix searx/static/themes/simple run build ) dump_return $? } themes.simple.test() { build_msg TEST "theme: simple" nodejs.ensure npm --prefix searx/static/themes/simple install npm --prefix searx/static/themes/simple run test dump_return $? } PYLINT_FILES=() while IFS= read -r line; do PYLINT_FILES+=("$line") done <<< "$(pylint.FILES)" # shellcheck disable=SC2119 main() { local _type local cmd="$1"; shift if [ "$cmd" == "" ]; then help err_msg "missing command" return 42 fi case "$cmd" in --getenv) var="$1"; echo "${!var}";; --help) help;; --*) help err_msg "unknown option $cmd" return 42 ;; *) _type="$(type -t "$cmd")" if [ "$_type" != 'function' ]; then err_msg "unknown command: $cmd / use --help" return 42 else "$cmd" "$@" fi ;; esac } main "$@"