[mod] load .config.sh from installation (utils/lib_install.sh)

**new** utils/lib_install.sh:
  Used to initialize installation procedures

  - Modified source_dot_config function that
    - loads .config.sh from an existing installation (at SEARX_SRC).
    - initialize **SEARX_SRC_INIT_FILES**
  - functions like:
    - install_log_searx_instance()
    - install_searx_get_state()

**modified** utils/searx.sh
    - obsolete environment SEARX_INSTANCE_NAME has been replaced
      by string 'SearXNG'.

**modified** utils/filtron.sh, utils/morty.sh, utils/searx.sh
  - source utils/lib_install.sh
  - normalize logging of environment variables using new function
    install_log_searx_instance()

**modified** utils/lib.sh
  - fix marginal typos

**Installation scripts**
  The utils/lib_install.sh is sourced by the installations scripts:

  - utils/searx.sh
  - utils/morty.sh
  - utils/filtron.sh

  If '${SEARX_SRC}/.config.sh' exists, the modified source_dot_config() function
  loads this configuration (instead of './.config.sh').

**SEARX_SRC_INIT_FILES**
  Array of file names to sync into a installation at $SEARX_SRC.  The file names
  are relative to the $REPO_ROOT.  Set by function init_SEARX_SRC_INIT_FILES().
  Most often theses are files like:

  - .config.sh
  - searx/settings.yml
  - utils/brand.env
  - ...

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

Makefile

@@ -63,6 +63,7 @@ test.shell:
 		utils/brand.env \
 		$(MTOOLS) \
 		utils/lib.sh \
+		utils/lib_install.sh \
 		utils/lib_static.sh \
 	        utils/filtron.sh \
 	        utils/searx.sh \

utils/filtron.sh

@@ -1,15 +1,11 @@
 #!/usr/bin/env bash
-# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-
 # SPDX-License-Identifier: AGPL-3.0-or-later
-# shellcheck disable=SC2119,SC2001
+# shellcheck disable=SC2001
 
 # shellcheck source=utils/lib.sh
 source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
-# shellcheck source=utils/brand.env
+# shellcheck source=utils/lib_install.sh
-source "${REPO_ROOT}/utils/brand.env"
+source "${REPO_ROOT}/utils/lib_install.sh"
-source_dot_config
-source "${REPO_ROOT}/utils/lxc-searx.env"
-in_container && lxc_set_suite_env
 
 # ----------------------------------------------------------------------------
 # config
@@ -94,28 +90,17 @@ apache (${PUBLIC_URL})
 nginx (${PUBLIC_URL})
   :install: nginx site with a reverse proxy (ProxyPass)
   :remove:  nginx site ${NGINX_FILTRON_SITE}
-
 filtron rules: ${FILTRON_RULES_TEMPLATE}
-
+---- sourced ${DOT_CONFIG} :
-If needed, set PUBLIC_URL of your WEB service in the '${DOT_CONFIG#"$REPO_ROOT/"}' file::
-  PUBLIC_URL     : ${PUBLIC_URL}
-  PUBLIC_HOST    : ${PUBLIC_HOST}
   SERVICE_USER        : ${SERVICE_USER}
+  SERVICE_HOME        : ${SERVICE_HOME}
   FILTRON_TARGET      : ${FILTRON_TARGET}
   FILTRON_API         : ${FILTRON_API}
   FILTRON_LISTEN      : ${FILTRON_LISTEN}
+  FILTRON_URL_PATH    : ${FILTRON_URL_PATH}
 EOF
-    if in_container; then
+
-        # in containers the service is listening on 0.0.0.0 (see lxc-searx.env)
+    install_log_searx_instance
-        for ip in $(global_IPs) ; do
-            if [[ $ip =~ .*:.* ]]; then
-                echo "  container URL (IPv6): http://[${ip#*|}]:4005/"
-            else
-                # IPv4:
-                echo "  container URL (IPv4): http://${ip#*|}:4005/"
-            fi
-        done
-    fi
     [[ -n ${1} ]] &&  err_msg "$1"
 }
 
@@ -349,16 +334,15 @@ inspect_service() {
 
     cat <<EOF
 
-sourced ${DOT_CONFIG#"$REPO_ROOT/"} :
+sourced ${DOT_CONFIG} :
-
+  SERVICE_USER        : ${SERVICE_USER}
-  PUBLIC_URL          : ${PUBLIC_URL}
+  SERVICE_HOME        : ${SERVICE_HOME}
-  PUBLIC_HOST         : ${PUBLIC_HOST}
+  FILTRON_TARGET      : ${FILTRON_TARGET}
-  FILTRON_URL_PATH    : ${FILTRON_URL_PATH}
   FILTRON_API         : ${FILTRON_API}
   FILTRON_LISTEN      : ${FILTRON_LISTEN}
-  FILTRON_TARGET      : ${FILTRON_TARGET}
+  FILTRON_URL_PATH    : ${FILTRON_URL_PATH}
-
 EOF
+    install_log_searx_instance
 
     if service_account_is_available "$SERVICE_USER"; then
         info_msg "service account $SERVICE_USER available."

utils/lib.sh

@@ -462,11 +462,11 @@ install_template() {
     fi
 
     if [[ -f "${dst}" ]] && cmp --silent "${template_file}" "${dst}" ; then
-        info_msg "file ${dst} allready installed"
+        info_msg "file ${dst} already installed"
         return 0
     fi
 
-    info_msg "diffrent file ${dst} allready exists on this host"
+    info_msg "different file ${dst} already exists on this host"
 
     while true; do
         choose_one _reply "choose next step with file $dst" \
@@ -1642,7 +1642,7 @@ git_clone() {
     #    git_clone <url> <path> [<branch> [<user>]]
     #
     #  First form uses $CACHE/<name> as destination folder, second form clones
-    #  into <path>.  If repository is allready cloned, pull from <branch> and
+    #  into <path>.  If repository is already cloned, pull from <branch> and
     #  update working tree (if needed, the caller has to stash local changes).
     #
     #    git clone https://github.com/searxng/searxng searx-src origin/master searxlogin

utils/lib_install.sh

@@ -0,0 +1,184 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+# https://github.com/koalaman/shellcheck/issues/356#issuecomment-853515285
+# shellcheck source=utils/lib.sh
+. /dev/null
+
+# Initialize installation procedures:
+#
+# - Modified source_dot_config function that
+#   - loads .config.sh from an existing installation (at SEARX_SRC).
+#   - initialize **SEARX_SRC_INIT_FILES**
+# - functions like:
+#   - install_log_searx_instance()
+#   - install_searx_get_state()
+#
+# usage:
+#   source lib_install.sh
+#
+# **Installation scripts**
+#
+# The utils/lib_install.sh is sourced by the installations scripts:
+#
+# - utils/searx.sh
+# - utils/morty.sh
+# - utils/filtron.sh
+#
+# If '${SEARX_SRC}/.config.sh' exists, the modified source_dot_config() function
+# loads this configuration (instead of './.config.sh').
+
+# **SEARX_SRC_INIT_FILES**
+#
+# Array of file names to sync into a installation at $SEARX_SRC.  The file names
+# are relative to the $REPO_ROOT.  Set by function init_SEARX_SRC_INIT_FILES().
+# Most often theses are files like:
+# - .config.sh
+# - searx/settings.yml
+# - utils/brand.env
+# - ...
+
+
+SEARX_SRC_INIT_FILES=()
+
+eval orig_"$(declare -f source_dot_config)"
+
+source_dot_config() {
+
+    # Modified source_dot_config function that
+    # - loads .config.sh from an existing installation (at SEARX_SRC).
+    # - initialize SEARX_SRC_INIT_FILES
+
+    if [ -z "$eval_SEARX_SRC" ]; then
+        export eval_SEARX_SRC='true'
+        SEARX_SRC=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARX_SRC)
+        if [ ! -r "${SEARX_SRC}" ]; then
+            build_msg INSTANCE "not yet cloned: ${SEARX_SRC}"
+            orig_source_dot_config
+            return 0
+        fi
+        build_msg INSTANCE "using instance at: ${SEARX_SRC}"
+
+        # set and log DOT_CONFIG
+        if [ -r "${SEARX_SRC}/.config.sh" ]; then
+            build_msg INSTANCE "switching to ${SEARX_SRC}/.config.sh"
+            DOT_CONFIG="${SEARX_SRC}/.config.sh"
+        else
+            build_msg INSTANCE "using local config: ${DOT_CONFIG}"
+        fi
+        init_SEARX_SRC_INIT_FILES
+    fi
+}
+
+init_SEARX_SRC_INIT_FILES(){
+    # init environment SEARX_SRC_INIT_FILES
+
+    # Monitor modified files in the working-tree from the local repository, only
+    # if the local file differs to the corresponding file in the instance.  Most
+    # often theses are files like:
+    #
+    #  - .config.sh
+    #  - searx/settings.yml
+    #  - utils/brand.env
+    #  - ...
+
+    # keep list empty if there is no installation
+    SEARX_SRC_INIT_FILES=()
+    if [ ! -r "$SEARX_SRC" ]; then
+        return 0
+    fi
+
+    local fname
+    local msg=""
+
+    # Monitor local modified files from the repository, only if the local file
+    # differs to the corresponding file in the instance
+
+    while IFS= read -r fname; do
+        if [ -z "$fname" ]; then
+            continue
+        fi
+        if [ -r "${SEARX_SRC}/${fname}" ]; then
+            # diff  "${REPO_ROOT}/${fname}" "${SEARX_SRC}/${fname}"
+            if ! cmp --silent "${REPO_ROOT}/${fname}" "${SEARX_SRC}/${fname}"; then
+                SEARX_SRC_INIT_FILES+=("${fname}")
+                build_msg INSTANCE "local clone (workingtree), modified file: ./$fname"
+                msg="to update use:  sudo -H ./utils/searx.sh install init-src"
+            fi
+        fi
+    done <<< "$(git diff --name-only)"
+    [ -n "$msg" ] &&  build_msg INSTANCE "$msg"
+}
+
+install_log_searx_instance() {
+
+    echo -e "---- SearXNG instance setup ${_BBlue}(status: $(install_searx_get_state))${_creset}"
+    echo -e "  SEARX_SETTINGS_PATH : ${_BBlue}${SEARX_SETTINGS_PATH}${_creset}"
+    echo -e "  SEARX_SRC           : ${_BBlue}${SEARX_SRC:-none}${_creset}"
+    echo -e "  SEARX_URL           : ${_BBlue}${SEARX_URL:-none}${_creset}"
+
+    if in_container; then
+        # searx is listening on 127.0.0.1 and not available from outside container
+        # in containers the service is listening on 0.0.0.0 (see lxc-searx.env)
+        echo -e "---- container setup"
+        echo -e "  ${_BBlack}HINT:${_creset} searx only listen on loopback device" \
+             "${_BBlack}inside${_creset} the container."
+        for ip in $(global_IPs) ; do
+            if [[ $ip =~ .*:.* ]]; then
+                echo "  container (IPv6): [${ip#*|}]"
+            else
+                # IPv4:
+                echo "  container (IPv4): ${ip#*|}"
+            fi
+        done
+    fi
+}
+
+install_searx_get_state(){
+
+    # usage: install_searx_get_state
+    #
+    # Prompts a string indicating the status of the installation procedure
+    #
+    # missing-searx-clone:
+    #    There is no clone at ${SEARX_SRC}
+    # missing-searx-pyenv:
+    #    There is no pyenv in ${SEARX_PYENV}
+    # installer-modified:
+    #    There are files modified locally in the installer (clone),
+    #    see ${SEARX_SRC_INIT_FILES} description.
+    # python-installed:
+    #    Scripts can be executed in instance's environment
+    #    - user:  ${SERVICE_USER}
+    #    - pyenv: ${SEARX_PYENV}
+
+    if ! [ -r "${SEARX_SRC}" ]; then
+        echo "missing-searx-clone"
+        return
+    fi
+    if ! [ -f "${SEARX_PYENV}/bin/activate" ]; then
+        echo "missing-searx-pyenv"
+        return
+    fi
+    if ! [ -r "${SEARX_SETTINGS_PATH}" ]; then
+        echo "missing-settings"
+        return
+    fi
+    if ! [ ${#SEARX_SRC_INIT_FILES[*]} -eq 0 ]; then
+        echo "installer-modified"
+        return
+    fi
+    echo "python-installed"
+}
+
+# Initialization of the installation procedure
+# --------------------------------------------
+
+# shellcheck source=utils/brand.env
+source "${REPO_ROOT}/utils/brand.env"
+
+source_dot_config
+
+# shellcheck source=utils/lxc-searx.env
+source "${REPO_ROOT}/utils/lxc-searx.env"
+in_container && lxc_set_suite_env

utils/morty.sh

@@ -1,15 +1,10 @@
 #!/usr/bin/env bash
-# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-
 # SPDX-License-Identifier: AGPL-3.0-or-later
 
 # shellcheck source=utils/lib.sh
 source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
-# shellcheck source=utils/brand.env
+# shellcheck source=utils/lib_install.sh
-source "${REPO_ROOT}/utils/brand.env"
+source "${REPO_ROOT}/utils/lib_install.sh"
-source_dot_config
-SEARX_URL="${PUBLIC_URL:-http://$(uname -n)/searx}"
-source "${REPO_ROOT}/utils/lxc-searx.env"
-in_container && lxc_set_suite_env
 
 # ----------------------------------------------------------------------------
 # config
@@ -86,12 +81,15 @@ apache : ${PUBLIC_URL_MORTY}
 nginx (${PUBLIC_URL_MORTY})
   :install: nginx site with a reverse proxy (ProxyPass)
   :remove:  nginx site ${NGINX_MORTY_SITE}
-
+----
-If needed, set the environment variables in the '${DOT_CONFIG#"$REPO_ROOT/"}' file::
+sourced ${DOT_CONFIG} :
-  PUBLIC_URL_MORTY:     ${PUBLIC_URL_MORTY}
-  MORTY_LISTEN:         ${MORTY_LISTEN}
   SERVICE_USER        : ${SERVICE_USER}
+  SERVICE_HOME        : ${SERVICE_HOME}
+  PUBLIC_URL_MORTY:   : ${PUBLIC_URL_MORTY}
+  MORTY_LISTEN:       : ${MORTY_LISTEN}
 EOF
+
+    install_log_searx_instance
     if in_container; then
         # in containers the service is listening on 0.0.0.0 (see lxc-searx.env)
         for ip in $(global_IPs) ; do
@@ -335,11 +333,14 @@ inspect_service() {
 
     cat <<EOF
 
-sourced ${DOT_CONFIG#"$REPO_ROOT/"} :
+sourced ${DOT_CONFIG} :
-
+  SERVICE_USER        : ${SERVICE_USER}
-  MORTY_LISTEN :   ${MORTY_LISTEN}
+  SERVICE_HOME        : ${SERVICE_HOME}
+  PUBLIC_URL_MORTY:   : ${PUBLIC_URL_MORTY}
+  MORTY_LISTEN:       : ${MORTY_LISTEN}
 
 EOF
+    install_log_searx_instance
 
     if service_account_is_available "$SERVICE_USER"; then
         info_msg "service account $SERVICE_USER available."
@@ -402,7 +403,7 @@ EOF
 }
 
 enable_debug() {
-    warn_msg "Do not enable debug in production enviroments!!"
+    warn_msg "Do not enable debug in production environments!!"
     info_msg "Enabling debug option needs to reinstall systemd service!"
     set_service_env_debug true
 }

utils/searx.sh

@@ -1,29 +1,23 @@
 #!/usr/bin/env bash
-# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-
 # SPDX-License-Identifier: AGPL-3.0-or-later
 # shellcheck disable=SC2001
 
 # shellcheck source=utils/lib.sh
 source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
-# shellcheck source=utils/brand.env
+
-source "${REPO_ROOT}/utils/brand.env"
+# shellcheck source=utils/lib_install.sh
-source_dot_config
+source "${REPO_ROOT}/utils/lib_install.sh"
-source "${REPO_ROOT}/utils/lxc-searx.env"
-in_container && lxc_set_suite_env
 
 # ----------------------------------------------------------------------------
 # config
 # ----------------------------------------------------------------------------
 
 PUBLIC_URL="${PUBLIC_URL:-http://$(uname -n)/searx}"
-
+SEARX_INTERNAL_HTTP="${SEARX_BIND_ADDRESS}:${SEARX_PORT}"
-SEARX_INTERNAL_HTTP="${SEARX_INTERNAL_HTTP:-127.0.0.1:8888}"
 
 SEARX_URL_PATH="${SEARX_URL_PATH:-$(echo "${PUBLIC_URL}" \
 | sed -e 's,^.*://[^/]*\(/.*\),\1,g')}"
 [[ "${SEARX_URL_PATH}" == "${PUBLIC_URL}" ]] && SEARX_URL_PATH=/
-SEARX_INSTANCE_NAME="${SEARX_INSTANCE_NAME:-searx@$(echo "$PUBLIC_URL" \
-| sed -e 's,^.*://\([^\:/]*\).*,\1,g') }"
 
 SERVICE_NAME="searx"
 SERVICE_USER="${SERVICE_USER:-${SERVICE_NAME}}"
@@ -182,29 +176,12 @@ option
 apache
   :install: apache site with the searx uwsgi app
   :remove:  apache site ${APACHE_FILTRON_SITE}
-
+---- sourced ${DOT_CONFIG}
-searx settings: ${SEARX_SETTINGS_PATH}
-
-If needed, set PUBLIC_URL of your WEB service in the '${DOT_CONFIG#"$REPO_ROOT/"}' file::
-  PUBLIC_URL          : ${PUBLIC_URL}
-  SEARX_INSTANCE_NAME : ${SEARX_INSTANCE_NAME}
   SERVICE_USER        : ${SERVICE_USER}
-  SEARX_INTERNAL_HTTP : http://${SEARX_INTERNAL_HTTP}
+  SERVICE_HOME        : ${SERVICE_HOME}
 EOF
-    if in_container; then
+
-        # searx is listening on 127.0.0.1 and not available from outside container
+    install_log_searx_instance
-        # in containers the service is listening on 0.0.0.0 (see lxc-searx.env)
-        echo -e "${_BBlack}HINT:${_creset} searx only listen on loopback device" \
-             "${_BBlack}inside${_creset} the container."
-        for ip in $(global_IPs) ; do
-            if [[ $ip =~ .*:.* ]]; then
-                echo "  container (IPv6): [${ip#*|}]"
-            else
-                # IPv4:
-                echo "  container (IPv4): ${ip#*|}"
-            fi
-        done
-    fi
     [[ -n ${1} ]] &&  err_msg "$1"
 }
 
@@ -231,7 +208,7 @@ main() {
                 *) usage "$_usage"; exit 42;;
             esac ;;
         install)
-            rst_title "$SEARX_INSTANCE_NAME" part
+            rst_title "SearXNG (install)" part
             sudo_or_exit
             case $2 in
                 all) install_all ;;
@@ -261,6 +238,7 @@ main() {
                 *) usage "$_usage"; exit 42;;
             esac ;;
         remove)
+            rst_title "SearXNG (remove)" part
             sudo_or_exit
             case $2 in
                 all) remove_all;;
@@ -307,7 +285,8 @@ main() {
 _service_prefix="  ${_Yellow}|$SERVICE_USER|${_creset} "
 
 install_all() {
-    rst_title "Install $SEARX_INSTANCE_NAME (service)"
+    rst_title "Install SearXNG (service)"
+    verify_continue_install
     pkg_install "$SEARX_PACKAGES"
     wait_key
     assert_user
@@ -348,13 +327,13 @@ EOF
 }
 
 remove_all() {
-    rst_title "De-Install $SEARX_INSTANCE_NAME (service)"
+    rst_title "De-Install SearXNG (service)"
 
     rst_para "\
 It goes without saying that this script can only be used to remove
 installations that were installed with this script."
 
-    if ! ask_yn "Do you really want to deinstall $SEARX_INSTANCE_NAME?"; then
+    if ! ask_yn "Do you really want to deinstall SearXNG?"; then
         return
     fi
     remove_searx_uwsgi
@@ -537,7 +516,6 @@ configure_searx() {
     tee_stderr 0.1 <<EOF | sudo -H -i 2>&1 |  prefix_stdout "$_service_prefix"
 cd ${SEARX_SRC}
 sed -i -e "s/ultrasecretkey/$(openssl rand -hex 16)/g" "$SEARX_SETTINGS_PATH"
-sed -i -e "s/{instance_name}/${SEARX_INSTANCE_NAME}/g" "$SEARX_SETTINGS_PATH"
 EOF
 }
 
@@ -577,14 +555,14 @@ remove_searx_uwsgi() {
 }
 
 activate_service() {
-    rst_title "Activate $SEARX_INSTANCE_NAME (service)" section
+    rst_title "Activate SearXNG (service)" section
     echo
     uWSGI_enable_app "$SEARX_UWSGI_APP"
     uWSGI_restart "$SEARX_UWSGI_APP"
 }
 
 deactivate_service() {
-    rst_title "De-Activate $SEARX_INSTANCE_NAME (service)" section
+    rst_title "De-Activate SearXNG (service)" section
     echo
     uWSGI_disable_app "$SEARX_UWSGI_APP"
     uWSGI_restart "$SEARX_UWSGI_APP"
@@ -609,7 +587,7 @@ EOF
 }
 
 enable_debug() {
-    warn_msg "Do not enable debug in production enviroments!!"
+    warn_msg "Do not enable debug in production environments!!"
     info_msg "try to enable debug mode ..."
     tee_stderr 0.1 <<EOF | sudo -H -i 2>&1 |  prefix_stdout "$_service_prefix"
 cd ${SEARX_SRC}
@@ -680,14 +658,11 @@ inspect_service() {
     rst_title "service status & log"
     cat <<EOF
 
-sourced ${DOT_CONFIG#"$REPO_ROOT/"} :
+sourced ${DOT_CONFIG} :
-
+  SERVICE_USER        : ${SERVICE_USER}
-  PUBLIC_URL          : ${PUBLIC_URL}
+  SERVICE_HOME        : ${SERVICE_HOME}
-  SEARX_URL_PATH      : ${SEARX_URL_PATH}
-  SEARX_INSTANCE_NAME : ${SEARX_INSTANCE_NAME}
-  SEARX_INTERNAL_HTTP  : ${SEARX_INTERNAL_HTTP}
-
 EOF
+    install_log_searx_instance
 
     if service_account_is_available "$SERVICE_USER"; then
         info_msg "Service account $SERVICE_USER exists."