From 1cfe7f2a7543b2994a1afd0d81da1962d04423b0 Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Tue, 3 Nov 2020 15:29:59 +0100 Subject: [PATCH 1/2] [enh] settings.yml: add use_default_settings option This change is backward compatible with the existing configurations. If a settings.yml loaded from an user defined location (SEARX_SETTINGS_PATH or /etc/searx/settings.yml), then this settings can relied on the default settings.yml with this option: user_default_settings:True --- Makefile | 17 +++++- docs/admin/settings.rst | 94 +++++++++++++++++++++++++++++++++ docs/blog/private-engines.rst | 2 + searx/__init__.py | 31 ++--------- searx/exceptions.py | 8 +++ searx/settings.py | 91 ++++++++++++++++++++++++++++++++ utils/update_user_settings.py | 98 +++++++++++++++++++++++++++++++++++ 7 files changed, 312 insertions(+), 29 deletions(-) create mode 100644 searx/settings.py create mode 100644 utils/update_user_settings.py diff --git a/Makefile b/Makefile index 29faaeefa..94309cefe 100644 --- a/Makefile +++ b/Makefile @@ -266,4 +266,19 @@ test.clean: travis.codecov: $(Q)$(PY_ENV_BIN)/python -m pip install codecov -.PHONY: $(PHONY) + +# user-settings +# ------------- + +PHONY += user-settings.create user-settings.update + +user-settings.update: pyenvinstall + $(Q)$(PY_ENV_ACT); pip install ruamel.yaml + $(Q)$(PY_ENV_ACT); python utils/update_user_settings.py ${SEARX_SETTINGS_PATH} + +user-settings.update.engines: pyenvinstall + $(Q)$(PY_ENV_ACT); pip install ruamel.yaml + $(Q)$(PY_ENV_ACT); python utils/update_user_settings.py --add-engines ${SEARX_SETTINGS_PATH} + + +.PHONY: $(PHONY) \ No newline at end of file diff --git a/docs/admin/settings.rst b/docs/admin/settings.rst index 58bce3541..cd944cc4c 100644 --- a/docs/admin/settings.rst +++ b/docs/admin/settings.rst @@ -206,3 +206,97 @@ Engine settings A few more options are possible, but they are pretty specific to some engines, and so won't be described here. + + +.. _settings location: + +settings.yml location +===================== + +First, searx will try to load settings.yml from these locations: + +1. the full path specified in the ``SEARX_SETTINGS_PATH`` environment variable. +2. ``/etc/searx/settings.yml`` + +If these files don't exist (or are empty or can't be read), searx uses the :origin:`searx/settings.yml` file. + +.. _ settings use_default_settings: + +use_default_settings +==================== + +.. note:: + + If searx is cloned from a git repository, most probably there is no need to have an user settings. + +The user defined settings.yml can relied on the default configuration :origin:`searx/settings.yml` using ``use_default_settings: True``. + +In the following example, the actual settings are the default settings defined in :origin:`searx/settings.yml` with the exception of the ``secret_key`` and the ``bind_address``: + +.. code-block:: yaml + + use_default_settings: true + server: + secret_key: "uvys6bRhKHUdFF5CqbJonSDSRN8H0sCBziNSrDGNVdpz7IeZhveVart3yvghoKHA" + server: + bind_address: "0.0.0.0" + +With ``use_default_settings: True``, each settings can be override in a similar way with one exception, the ``engines`` section: + +* If the ``engines`` section is not defined in the user settings, searx uses the engines from the default setttings (the above example). +* If the ``engines`` section is defined then: + + * searx loads only the engines declare in the user setttings. + * searx merges the configuration according to the engine name. + +In the following example, only three engines are available. Each engine configuration is merged with the default configuration. + +.. code-block:: yaml + + use_default_settings: true + server: + secret_key: "uvys6bRhKHUdFF5CqbJonSDSRN8H0sCBziNSrDGNVdpz7IeZhveVart3yvghoKHA" + engines: + - name: wikipedia + - name: wikidata + - name: ddg definitions + +Another example where four engines are available. The arch linux wiki engine has a :ref:`token`. + +.. code-block:: yaml + + use_default_settings: true + server: + secret_key: "uvys6bRhKHUdFF5CqbJonSDSRN8H0sCBziNSrDGNVdpz7IeZhveVart3yvghoKHA" + engines: + - name: arch linux wiki + tokens: ['$ecretValue'] + - name: wikipedia + - name: wikidata + - name: ddg definitions + +automatic update +---------------- + +The following comand creates or updates a minimal user settings (a secret key is defined if it is not already the case): + +.. code-block:: sh + + make SEARX_SETTINGS_PATH=/etc/searx/settings.yml user-settings.update + +Set ``SEARX_SETTINGS_PATH`` to your user settings path. + +As soon the user settings contains an ``engines`` section, it becomes difficult to keep the engine list updated. +The following command creates or updates the user settings including the ``engines`` section: + +.. code-block:: sh + + make SEARX_SETTINGS_PATH=/etc/searx/settings.yml user-settings.update.engines + +After that ``/etc/searx/settings.yml`` + +* has a ``secret key`` +* has a ``engine`` section if it is not already the case, moreover the command: + + * has deleted engines that do not exist in the default settings. + * has added engines that exist in the default settings but are not declare in the user settings. diff --git a/docs/blog/private-engines.rst b/docs/blog/private-engines.rst index 796f0fc69..027cc3dd3 100644 --- a/docs/blog/private-engines.rst +++ b/docs/blog/private-engines.rst @@ -7,6 +7,8 @@ enabled engines on their instances. It might be because they do not want to expose some private information through an offline engine. Or they would rather share engines only with their trusted friends or colleagues. +.. _private engines: + Private engines =============== diff --git a/searx/__init__.py b/searx/__init__.py index 887ef806d..214e554d4 100644 --- a/searx/__init__.py +++ b/searx/__init__.py @@ -16,39 +16,15 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >. ''' import logging +import searx.settings from os import environ from os.path import realpath, dirname, join, abspath, isfile -from io import open -from yaml import safe_load searx_dir = abspath(dirname(__file__)) engine_dir = dirname(realpath(__file__)) static_path = abspath(join(dirname(__file__), 'static')) - - -def check_settings_yml(file_name): - if isfile(file_name): - return file_name - else: - return None - - -# find location of settings.yml -if 'SEARX_SETTINGS_PATH' in environ: - # if possible set path to settings using the - # enviroment variable SEARX_SETTINGS_PATH - settings_path = check_settings_yml(environ['SEARX_SETTINGS_PATH']) -else: - # if not, get it from searx code base or last solution from /etc/searx - settings_path = check_settings_yml(join(searx_dir, 'settings.yml')) or check_settings_yml('/etc/searx/settings.yml') - -if not settings_path: - raise Exception('settings.yml not found') - -# load settings -with open(settings_path, 'r', encoding='utf-8') as settings_yaml: - settings = safe_load(settings_yaml) +settings, settings_load_message = searx.settings.load_settings() if settings['ui']['static_path']: static_path = settings['ui']['static_path'] @@ -58,7 +34,6 @@ enable debug if the environnement variable SEARX_DEBUG is 1 or true (whatever the value in settings.yml) or general.debug=True in settings.yml - disable debug if the environnement variable SEARX_DEBUG is 0 or false (whatever the value in settings.yml) @@ -78,7 +53,7 @@ else: logging.basicConfig(level=logging.WARNING) logger = logging.getLogger('searx') -logger.debug('read configuration from %s', settings_path) +logger.info(settings_load_message) logger.info('Initialisation done') if 'SEARX_SECRET' in environ: diff --git a/searx/exceptions.py b/searx/exceptions.py index 4af816272..2d1b1167e 100644 --- a/searx/exceptions.py +++ b/searx/exceptions.py @@ -31,3 +31,11 @@ class SearxParameterException(SearxException): self.message = message self.parameter_name = name self.parameter_value = value + + +class SearxSettingsException(SearxException): + + def __init__(self, message, filename): + super().__init__(message) + self.message = message + self.filename = filename diff --git a/searx/settings.py b/searx/settings.py new file mode 100644 index 000000000..cdddff589 --- /dev/null +++ b/searx/settings.py @@ -0,0 +1,91 @@ +import collections.abc + +import yaml +from searx.exceptions import SearxSettingsException +from os import environ +from os.path import dirname, join, abspath, isfile + + +searx_dir = abspath(dirname(__file__)) + + +def check_settings_yml(file_name): + if isfile(file_name): + return file_name + else: + return None + + +def load_yaml(file_name): + try: + with open(file_name, 'r', encoding='utf-8') as settings_yaml: + settings = yaml.safe_load(settings_yaml) + if not isinstance(settings, dict) or len(settings) == 0: + raise SearxSettingsException('Empty file', file_name) + return settings + except IOError as e: + raise SearxSettingsException(e, file_name) + except yaml.YAMLError as e: + raise SearxSettingsException(e, file_name) + + +def get_default_settings_path(): + return check_settings_yml(join(searx_dir, 'settings.yml')) + + +def get_user_settings_path(): + # find location of settings.yml + if 'SEARX_SETTINGS_PATH' in environ: + # if possible set path to settings using the + # enviroment variable SEARX_SETTINGS_PATH + return check_settings_yml(environ['SEARX_SETTINGS_PATH']) + else: + # if not, get it from searx code base or last solution from /etc/searx + return check_settings_yml('/etc/searx/settings.yml') + + +def update_dict(d, u): + for k, v in u.items(): + if isinstance(v, collections.abc.Mapping): + d[k] = update_dict(d.get(k, {}), v) + else: + d[k] = v + return d + + +def update_settings(default_settings, user_settings): + for k, v in user_settings.items(): + if k == 'use_default_settings': + continue + elif k == 'engines': + default_engines = default_settings[k] + default_engines_dict = dict((definition['name'], definition) for definition in default_engines) + default_settings[k] = [update_dict(default_engines_dict[definition['name']], definition) + for definition in v] + else: + update_dict(default_settings[k], v) + + return default_settings + + +def load_settings(load_user_setttings=True): + default_settings_path = get_default_settings_path() + user_settings_path = get_user_settings_path() + if user_settings_path is None or not load_user_setttings: + # no user settings + return (load_yaml(default_settings_path), + 'load the default settings from {}'.format(default_settings_path)) + + # user settings + user_settings = load_yaml(user_settings_path) + if user_settings.get('use_default_settings'): + # the user settings are merged with the default configuration + default_settings = load_yaml(default_settings_path) + update_settings(default_settings, user_settings) + return (default_settings, + 'merge the default settings ( {} ) and the user setttings ( {} )' + .format(default_settings_path, user_settings_path)) + + # the user settings, fully replace the default configuration + return (user_settings, + 'load the user settings from {}'.format(user_settings_path)) diff --git a/utils/update_user_settings.py b/utils/update_user_settings.py new file mode 100644 index 000000000..fb6fd0b3f --- /dev/null +++ b/utils/update_user_settings.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python + +# set path +from sys import path +from os.path import realpath, dirname, join +path.append(realpath(dirname(realpath(__file__)) + '/../')) + +import argparse +import sys +import string +import ruamel.yaml +import secrets +import collections +from ruamel.yaml.scalarstring import SingleQuotedScalarString, DoubleQuotedScalarString +from searx.settings import load_settings, check_settings_yml, get_default_settings_path +from searx.exceptions import SearxSettingsException + + +RANDOM_STRING_LETTERS = string.ascii_lowercase + string.digits + string.ascii_uppercase + + +def get_random_string(): + r = [secrets.choice(RANDOM_STRING_LETTERS) for _ in range(64)] + return ''.join(r) + + +def main(prog_arg): + yaml = ruamel.yaml.YAML() + yaml.preserve_quotes = True + yaml.indent(mapping=4, sequence=1, offset=2) + user_settings_path = prog_args.get('user-settings-yaml') + + try: + default_settings, _ = load_settings(False) + if check_settings_yml(user_settings_path): + with open(user_settings_path, 'r', encoding='utf-8') as f: + user_settings = yaml.load(f.read()) + new_user_settings = False + else: + user_settings = yaml.load('use_default_settings: True') + new_user_settings = True + except SearxSettingsException as e: + sys.stderr.write(str(e)) + return + + if not new_user_settings and not user_settings.get('use_default_settings'): + sys.stderr.write('settings.yml already exists and use_default_settings is not True') + return + + user_settings['use_default_settings'] = True + use_default_settings_comment = "settings based on " + get_default_settings_path() + user_settings.yaml_add_eol_comment(use_default_settings_comment, 'use_default_settings') + + if user_settings.get('server', {}).get('secret_key') in [None, 'ultrasecretkey']: + user_settings.setdefault('server', {})['secret_key'] = DoubleQuotedScalarString(get_random_string()) + + user_engines = user_settings.get('engines') + if user_engines: + has_user_engines = True + user_engines_dict = dict((definition['name'], definition) for definition in user_engines) + else: + has_user_engines = False + user_engines_dict = {} + user_engines = [] + + # remove old engines + if prog_arg.get('add-engines') or has_user_engines: + default_engines_dict = dict((definition['name'], definition) for definition in default_settings['engines']) + for i, engine in enumerate(user_engines): + if engine['name'] not in default_engines_dict: + del user_engines[i] + + # add new engines + if prog_arg.get('add-engines'): + for engine in default_settings.get('engines', {}): + if engine['name'] not in user_engines_dict: + user_engines.append({'name': engine['name']}) + user_settings['engines'] = user_engines + + # output + if prog_arg.get('dry-run'): + yaml.dump(user_settings, sys.stdout) + else: + with open(user_settings_path, 'w', encoding='utf-8') as f: + yaml.dump(user_settings, f) + + +def parse_args(): + parser = argparse.ArgumentParser(description='Update user settings.yml') + parser.add_argument('--add-engines', dest='add-engines', default=False, action='store_true', help='Add new engines') + parser.add_argument('--dry-run', dest='dry-run', default=False, action='store_true', help='Dry run') + parser.add_argument('user-settings-yaml', type=str) + return vars(parser.parse_args()) + + +if __name__ == '__main__': + prog_args = parse_args() + main(prog_args) From b4b81a5e1a74f03926e3c2e6f8c5fade99f7eabb Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Fri, 27 Nov 2020 19:32:45 +0100 Subject: [PATCH 2/2] [enh] settings.yml: add use_default_settings option (2nd version) --- Makefile | 17 +-- docs/admin/settings.rst | 75 ++++------ searx/__init__.py | 4 +- searx/settings.py | 91 ------------ searx/settings_loader.py | 129 ++++++++++++++++++ tests/unit/settings/empty_settings.yml | 0 tests/unit/settings/syntaxerror_settings.yml | 2 + tests/unit/settings/user_settings.yml | 111 +++++++++++++++ .../unit/settings/user_settings_keep_only.yml | 14 ++ tests/unit/settings/user_settings_remove.yml | 10 ++ tests/unit/settings/user_settings_remove2.yml | 15 ++ tests/unit/settings/user_settings_simple.yml | 6 + tests/unit/test_settings_loader.py | 122 +++++++++++++++++ utils/update_user_settings.py | 98 ------------- 14 files changed, 441 insertions(+), 253 deletions(-) delete mode 100644 searx/settings.py create mode 100644 searx/settings_loader.py create mode 100644 tests/unit/settings/empty_settings.yml create mode 100644 tests/unit/settings/syntaxerror_settings.yml create mode 100644 tests/unit/settings/user_settings.yml create mode 100644 tests/unit/settings/user_settings_keep_only.yml create mode 100644 tests/unit/settings/user_settings_remove.yml create mode 100644 tests/unit/settings/user_settings_remove2.yml create mode 100644 tests/unit/settings/user_settings_simple.yml create mode 100644 tests/unit/test_settings_loader.py delete mode 100644 utils/update_user_settings.py diff --git a/Makefile b/Makefile index 94309cefe..29faaeefa 100644 --- a/Makefile +++ b/Makefile @@ -266,19 +266,4 @@ test.clean: travis.codecov: $(Q)$(PY_ENV_BIN)/python -m pip install codecov - -# user-settings -# ------------- - -PHONY += user-settings.create user-settings.update - -user-settings.update: pyenvinstall - $(Q)$(PY_ENV_ACT); pip install ruamel.yaml - $(Q)$(PY_ENV_ACT); python utils/update_user_settings.py ${SEARX_SETTINGS_PATH} - -user-settings.update.engines: pyenvinstall - $(Q)$(PY_ENV_ACT); pip install ruamel.yaml - $(Q)$(PY_ENV_ACT); python utils/update_user_settings.py --add-engines ${SEARX_SETTINGS_PATH} - - -.PHONY: $(PHONY) \ No newline at end of file +.PHONY: $(PHONY) diff --git a/docs/admin/settings.rst b/docs/admin/settings.rst index cd944cc4c..532b99752 100644 --- a/docs/admin/settings.rst +++ b/docs/admin/settings.rst @@ -235,68 +235,51 @@ In the following example, the actual settings are the default settings defined i .. code-block:: yaml - use_default_settings: true + use_default_settings: True server: secret_key: "uvys6bRhKHUdFF5CqbJonSDSRN8H0sCBziNSrDGNVdpz7IeZhveVart3yvghoKHA" - server: bind_address: "0.0.0.0" -With ``use_default_settings: True``, each settings can be override in a similar way with one exception, the ``engines`` section: +With ``use_default_settings: True``, each settings can be override in a similar way, the ``engines`` section is merged according to the engine ``name``. -* If the ``engines`` section is not defined in the user settings, searx uses the engines from the default setttings (the above example). -* If the ``engines`` section is defined then: - - * searx loads only the engines declare in the user setttings. - * searx merges the configuration according to the engine name. - -In the following example, only three engines are available. Each engine configuration is merged with the default configuration. +In this example, searx will load all the engine and the arch linux wiki engine has a :ref:`token`: .. code-block:: yaml - use_default_settings: true - server: - secret_key: "uvys6bRhKHUdFF5CqbJonSDSRN8H0sCBziNSrDGNVdpz7IeZhveVart3yvghoKHA" - engines: - - name: wikipedia - - name: wikidata - - name: ddg definitions - -Another example where four engines are available. The arch linux wiki engine has a :ref:`token`. - -.. code-block:: yaml - - use_default_settings: true + use_default_settings: True server: secret_key: "uvys6bRhKHUdFF5CqbJonSDSRN8H0sCBziNSrDGNVdpz7IeZhveVart3yvghoKHA" engines: - name: arch linux wiki tokens: ['$ecretValue'] - - name: wikipedia - - name: wikidata - - name: ddg definitions -automatic update ----------------- +It is possible to remove some engines from the default settings. The following example is similar to the above one, but searx doesn't load the the google engine: -The following comand creates or updates a minimal user settings (a secret key is defined if it is not already the case): +.. code-block:: yaml -.. code-block:: sh + use_default_settings: + engines: + remove: + - google + server: + secret_key: "uvys6bRhKHUdFF5CqbJonSDSRN8H0sCBziNSrDGNVdpz7IeZhveVart3yvghoKHA" + engines: + - name: arch linux wiki + tokens: ['$ecretValue'] - make SEARX_SETTINGS_PATH=/etc/searx/settings.yml user-settings.update +As an alternative, it is possible to specify the engines to keep. In the following example, searx has only two engines: -Set ``SEARX_SETTINGS_PATH`` to your user settings path. +.. code-block:: yaml -As soon the user settings contains an ``engines`` section, it becomes difficult to keep the engine list updated. -The following command creates or updates the user settings including the ``engines`` section: - -.. code-block:: sh - - make SEARX_SETTINGS_PATH=/etc/searx/settings.yml user-settings.update.engines - -After that ``/etc/searx/settings.yml`` - -* has a ``secret key`` -* has a ``engine`` section if it is not already the case, moreover the command: - - * has deleted engines that do not exist in the default settings. - * has added engines that exist in the default settings but are not declare in the user settings. + use_default_settings: + engines: + keep_only: + - google + - duckduckgo + server: + secret_key: "uvys6bRhKHUdFF5CqbJonSDSRN8H0sCBziNSrDGNVdpz7IeZhveVart3yvghoKHA" + engines: + - name: google + tokens: ['$ecretValue'] + - name: duckduckgo + tokens: ['$ecretValue'] diff --git a/searx/__init__.py b/searx/__init__.py index 214e554d4..9bbc7c8c3 100644 --- a/searx/__init__.py +++ b/searx/__init__.py @@ -16,7 +16,7 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >. ''' import logging -import searx.settings +import searx.settings_loader from os import environ from os.path import realpath, dirname, join, abspath, isfile @@ -24,7 +24,7 @@ from os.path import realpath, dirname, join, abspath, isfile searx_dir = abspath(dirname(__file__)) engine_dir = dirname(realpath(__file__)) static_path = abspath(join(dirname(__file__), 'static')) -settings, settings_load_message = searx.settings.load_settings() +settings, settings_load_message = searx.settings_loader.load_settings() if settings['ui']['static_path']: static_path = settings['ui']['static_path'] diff --git a/searx/settings.py b/searx/settings.py deleted file mode 100644 index cdddff589..000000000 --- a/searx/settings.py +++ /dev/null @@ -1,91 +0,0 @@ -import collections.abc - -import yaml -from searx.exceptions import SearxSettingsException -from os import environ -from os.path import dirname, join, abspath, isfile - - -searx_dir = abspath(dirname(__file__)) - - -def check_settings_yml(file_name): - if isfile(file_name): - return file_name - else: - return None - - -def load_yaml(file_name): - try: - with open(file_name, 'r', encoding='utf-8') as settings_yaml: - settings = yaml.safe_load(settings_yaml) - if not isinstance(settings, dict) or len(settings) == 0: - raise SearxSettingsException('Empty file', file_name) - return settings - except IOError as e: - raise SearxSettingsException(e, file_name) - except yaml.YAMLError as e: - raise SearxSettingsException(e, file_name) - - -def get_default_settings_path(): - return check_settings_yml(join(searx_dir, 'settings.yml')) - - -def get_user_settings_path(): - # find location of settings.yml - if 'SEARX_SETTINGS_PATH' in environ: - # if possible set path to settings using the - # enviroment variable SEARX_SETTINGS_PATH - return check_settings_yml(environ['SEARX_SETTINGS_PATH']) - else: - # if not, get it from searx code base or last solution from /etc/searx - return check_settings_yml('/etc/searx/settings.yml') - - -def update_dict(d, u): - for k, v in u.items(): - if isinstance(v, collections.abc.Mapping): - d[k] = update_dict(d.get(k, {}), v) - else: - d[k] = v - return d - - -def update_settings(default_settings, user_settings): - for k, v in user_settings.items(): - if k == 'use_default_settings': - continue - elif k == 'engines': - default_engines = default_settings[k] - default_engines_dict = dict((definition['name'], definition) for definition in default_engines) - default_settings[k] = [update_dict(default_engines_dict[definition['name']], definition) - for definition in v] - else: - update_dict(default_settings[k], v) - - return default_settings - - -def load_settings(load_user_setttings=True): - default_settings_path = get_default_settings_path() - user_settings_path = get_user_settings_path() - if user_settings_path is None or not load_user_setttings: - # no user settings - return (load_yaml(default_settings_path), - 'load the default settings from {}'.format(default_settings_path)) - - # user settings - user_settings = load_yaml(user_settings_path) - if user_settings.get('use_default_settings'): - # the user settings are merged with the default configuration - default_settings = load_yaml(default_settings_path) - update_settings(default_settings, user_settings) - return (default_settings, - 'merge the default settings ( {} ) and the user setttings ( {} )' - .format(default_settings_path, user_settings_path)) - - # the user settings, fully replace the default configuration - return (user_settings, - 'load the user settings from {}'.format(user_settings_path)) diff --git a/searx/settings_loader.py b/searx/settings_loader.py new file mode 100644 index 000000000..172069bd5 --- /dev/null +++ b/searx/settings_loader.py @@ -0,0 +1,129 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later + +from os import environ +from os.path import dirname, join, abspath, isfile +from collections.abc import Mapping +from itertools import filterfalse + +import yaml + +from searx.exceptions import SearxSettingsException + + +searx_dir = abspath(dirname(__file__)) + + +def check_settings_yml(file_name): + if isfile(file_name): + return file_name + return None + + +def load_yaml(file_name): + try: + with open(file_name, 'r', encoding='utf-8') as settings_yaml: + return yaml.safe_load(settings_yaml) + except IOError as e: + raise SearxSettingsException(e, file_name) + except yaml.YAMLError as e: + raise SearxSettingsException(e, file_name) + + +def get_default_settings_path(): + return check_settings_yml(join(searx_dir, 'settings.yml')) + + +def get_user_settings_path(): + # find location of settings.yml + if 'SEARX_SETTINGS_PATH' in environ: + # if possible set path to settings using the + # enviroment variable SEARX_SETTINGS_PATH + return check_settings_yml(environ['SEARX_SETTINGS_PATH']) + + # if not, get it from searx code base or last solution from /etc/searx + return check_settings_yml('/etc/searx/settings.yml') + + +def update_dict(default_dict, user_dict): + for k, v in user_dict.items(): + if isinstance(v, Mapping): + default_dict[k] = update_dict(default_dict.get(k, {}), v) + else: + default_dict[k] = v + return default_dict + + +def update_settings(default_settings, user_settings): + # merge everything except the engines + for k, v in user_settings.items(): + if k not in ('use_default_settings', 'engines'): + update_dict(default_settings[k], v) + + # parse the engines + remove_engines = None + keep_only_engines = None + use_default_settings = user_settings.get('use_default_settings') + if isinstance(use_default_settings, dict): + remove_engines = use_default_settings.get('engines', {}).get('remove') + keep_only_engines = use_default_settings.get('engines', {}).get('keep_only') + + if 'engines' in user_settings or remove_engines is not None or keep_only_engines is not None: + engines = default_settings['engines'] + + # parse "use_default_settings.engines.remove" + if remove_engines is not None: + engines = list(filterfalse(lambda engine: (engine.get('name')) in remove_engines, engines)) + + # parse "use_default_settings.engines.keep_only" + if keep_only_engines is not None: + engines = list(filter(lambda engine: (engine.get('name')) in keep_only_engines, engines)) + + # parse "engines" + user_engines = user_settings.get('engines') + if user_engines: + engines_dict = dict((definition['name'], definition) for definition in engines) + for user_engine in user_engines: + default_engine = engines_dict.get(user_engine['name']) + if default_engine: + update_dict(default_engine, user_engine) + else: + engines.append(user_engine) + + # store the result + default_settings['engines'] = engines + + return default_settings + + +def is_use_default_settings(user_settings): + use_default_settings = user_settings.get('use_default_settings') + if use_default_settings is True: + return True + if isinstance(use_default_settings, dict): + return True + if use_default_settings is False or use_default_settings is None: + return False + raise ValueError('Invalid value for use_default_settings') + + +def load_settings(load_user_setttings=True): + default_settings_path = get_default_settings_path() + user_settings_path = get_user_settings_path() + if user_settings_path is None or not load_user_setttings: + # no user settings + return (load_yaml(default_settings_path), + 'load the default settings from {}'.format(default_settings_path)) + + # user settings + user_settings = load_yaml(user_settings_path) + if is_use_default_settings(user_settings): + # the user settings are merged with the default configuration + default_settings = load_yaml(default_settings_path) + update_settings(default_settings, user_settings) + return (default_settings, + 'merge the default settings ( {} ) and the user setttings ( {} )' + .format(default_settings_path, user_settings_path)) + + # the user settings, fully replace the default configuration + return (user_settings, + 'load the user settings from {}'.format(user_settings_path)) diff --git a/tests/unit/settings/empty_settings.yml b/tests/unit/settings/empty_settings.yml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/unit/settings/syntaxerror_settings.yml b/tests/unit/settings/syntaxerror_settings.yml new file mode 100644 index 000000000..6d3b0f9a0 --- /dev/null +++ b/tests/unit/settings/syntaxerror_settings.yml @@ -0,0 +1,2 @@ +Test: + ********** diff --git a/tests/unit/settings/user_settings.yml b/tests/unit/settings/user_settings.yml new file mode 100644 index 000000000..f5b6c7173 --- /dev/null +++ b/tests/unit/settings/user_settings.yml @@ -0,0 +1,111 @@ +general: + debug : False + instance_name : "searx" + +search: + safe_search : 0 + autocomplete : "" + default_lang : "" + ban_time_on_fail : 5 + max_ban_time_on_fail : 120 + +server: + port : 9000 + bind_address : "0.0.0.0" + secret_key : "user_settings_secret" + base_url : False + image_proxy : False + http_protocol_version : "1.0" + method: "POST" + default_http_headers: + X-Content-Type-Options : nosniff + X-XSS-Protection : 1; mode=block + X-Download-Options : noopen + X-Robots-Tag : noindex, nofollow + Referrer-Policy : no-referrer + +ui: + static_path : "" + templates_path : "" + default_theme : oscar + default_locale : "" + theme_args : + oscar_style : logicodev + +engines: + - name : wikidata + engine : wikidata + shortcut : wd + timeout : 3.0 + weight : 2 + + - name : wikibooks + engine : mediawiki + shortcut : wb + categories : general + base_url : "https://{language}.wikibooks.org/" + number_of_results : 5 + search_type : text + + - name : wikinews + engine : mediawiki + shortcut : wn + categories : news + base_url : "https://{language}.wikinews.org/" + number_of_results : 5 + search_type : text + + - name : wikiquote + engine : mediawiki + shortcut : wq + categories : general + base_url : "https://{language}.wikiquote.org/" + number_of_results : 5 + search_type : text + +locales: + en : English + ar : العَرَبِيَّة (Arabic) + bg : Български (Bulgarian) + bo : བོད་སྐད་ (Tibetian) + ca : Català (Catalan) + cs : Čeština (Czech) + cy : Cymraeg (Welsh) + da : Dansk (Danish) + de : Deutsch (German) + el_GR : Ελληνικά (Greek_Greece) + eo : Esperanto (Esperanto) + es : Español (Spanish) + et : Eesti (Estonian) + eu : Euskara (Basque) + fa_IR : (fārsī) فارسى (Persian) + fi : Suomi (Finnish) + fil : Wikang Filipino (Filipino) + fr : Français (French) + gl : Galego (Galician) + he : עברית (Hebrew) + hr : Hrvatski (Croatian) + hu : Magyar (Hungarian) + ia : Interlingua (Interlingua) + it : Italiano (Italian) + ja : 日本語 (Japanese) + lt : Lietuvių (Lithuanian) + nl : Nederlands (Dutch) + nl_BE : Vlaams (Dutch_Belgium) + oc : Lenga D'òc (Occitan) + pl : Polski (Polish) + pt : Português (Portuguese) + pt_BR : Português (Portuguese_Brazil) + ro : Română (Romanian) + ru : Русский (Russian) + sk : Slovenčina (Slovak) + sl : Slovenski (Slovene) + sr : српски (Serbian) + sv : Svenska (Swedish) + te : తెలుగు (telugu) + ta : தமிழ் (Tamil) + tr : Türkçe (Turkish) + uk : українська мова (Ukrainian) + vi : tiếng việt (Vietnamese) + zh : 中文 (Chinese) + zh_TW : 國語 (Taiwanese Mandarin) diff --git a/tests/unit/settings/user_settings_keep_only.yml b/tests/unit/settings/user_settings_keep_only.yml new file mode 100644 index 000000000..518f18bde --- /dev/null +++ b/tests/unit/settings/user_settings_keep_only.yml @@ -0,0 +1,14 @@ +use_default_settings: + engines: + keep_only: + - wikibooks + - wikinews +server: + secret_key: "user_secret_key" + bind_address: "0.0.0.0" + default_http_headers: + Custom-Header: Custom-Value +engines: + - name: wikipedia + - name: newengine + engine: dummy diff --git a/tests/unit/settings/user_settings_remove.yml b/tests/unit/settings/user_settings_remove.yml new file mode 100644 index 000000000..c4fd85df7 --- /dev/null +++ b/tests/unit/settings/user_settings_remove.yml @@ -0,0 +1,10 @@ +use_default_settings: + engines: + remove: + - wikibooks + - wikinews +server: + secret_key: "user_secret_key" + bind_address: "0.0.0.0" + default_http_headers: + Custom-Header: Custom-Value diff --git a/tests/unit/settings/user_settings_remove2.yml b/tests/unit/settings/user_settings_remove2.yml new file mode 100644 index 000000000..e9be325dc --- /dev/null +++ b/tests/unit/settings/user_settings_remove2.yml @@ -0,0 +1,15 @@ +use_default_settings: + engines: + remove: + - wikibooks + - wikinews +server: + secret_key: "user_secret_key" + bind_address: "0.0.0.0" + default_http_headers: + Custom-Header: Custom-Value +engines: + - name: wikipedia + tokens: ['secret_token'] + - name: newengine + engine: dummy diff --git a/tests/unit/settings/user_settings_simple.yml b/tests/unit/settings/user_settings_simple.yml new file mode 100644 index 000000000..36e5f1647 --- /dev/null +++ b/tests/unit/settings/user_settings_simple.yml @@ -0,0 +1,6 @@ +use_default_settings: True +server: + secret_key: "user_secret_key" + bind_address: "0.0.0.0" + default_http_headers: + Custom-Header: Custom-Value diff --git a/tests/unit/test_settings_loader.py b/tests/unit/test_settings_loader.py new file mode 100644 index 000000000..7df64e524 --- /dev/null +++ b/tests/unit/test_settings_loader.py @@ -0,0 +1,122 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later + +from os.path import dirname, join, abspath +from unittest.mock import patch + +from searx.testing import SearxTestCase +from searx.exceptions import SearxSettingsException +from searx import settings_loader + + +test_dir = abspath(dirname(__file__)) + + +class TestLoad(SearxTestCase): + + def test_load_zero(self): + with self.assertRaises(SearxSettingsException): + settings_loader.load_yaml('/dev/zero') + + with self.assertRaises(SearxSettingsException): + settings_loader.load_yaml(join(test_dir, '/settings/syntaxerror_settings.yml')) + + with self.assertRaises(SearxSettingsException): + settings_loader.load_yaml(join(test_dir, '/settings/empty_settings.yml')) + + def test_check_settings_yml(self): + self.assertIsNone(settings_loader.check_settings_yml('/dev/zero')) + + bad_settings_path = join(test_dir, 'settings/syntaxerror_settings.yml') + self.assertEqual(settings_loader.check_settings_yml(bad_settings_path), bad_settings_path) + + +class TestDefaultSettings(SearxTestCase): + + def test_load(self): + settings, msg = settings_loader.load_settings(load_user_setttings=False) + self.assertTrue(msg.startswith('load the default settings from')) + self.assertFalse(settings['general']['debug']) + self.assertTrue(isinstance(settings['general']['instance_name'], str)) + self.assertEqual(settings['server']['secret_key'], "ultrasecretkey") + self.assertTrue(isinstance(settings['server']['port'], int)) + self.assertTrue(isinstance(settings['server']['bind_address'], str)) + self.assertTrue(isinstance(settings['engines'], list)) + self.assertTrue(isinstance(settings['locales'], dict)) + self.assertTrue(isinstance(settings['doi_resolvers'], dict)) + self.assertTrue(isinstance(settings['default_doi_resolver'], str)) + + +class TestUserSettings(SearxTestCase): + + def test_is_use_default_settings(self): + self.assertFalse(settings_loader.is_use_default_settings({})) + self.assertTrue(settings_loader.is_use_default_settings({'use_default_settings': True})) + self.assertTrue(settings_loader.is_use_default_settings({'use_default_settings': {}})) + with self.assertRaises(ValueError): + self.assertFalse(settings_loader.is_use_default_settings({'use_default_settings': 1})) + with self.assertRaises(ValueError): + self.assertFalse(settings_loader.is_use_default_settings({'use_default_settings': 0})) + + def test_user_settings_not_found(self): + with patch.dict(settings_loader.environ, + {'SEARX_SETTINGS_PATH': '/dev/null'}): + settings, msg = settings_loader.load_settings() + self.assertTrue(msg.startswith('load the default settings from')) + self.assertEqual(settings['server']['secret_key'], "ultrasecretkey") + + def test_user_settings(self): + with patch.dict(settings_loader.environ, + {'SEARX_SETTINGS_PATH': join(test_dir, 'settings/user_settings_simple.yml')}): + settings, msg = settings_loader.load_settings() + self.assertTrue(msg.startswith('merge the default settings')) + self.assertEqual(settings['server']['secret_key'], "user_secret_key") + self.assertEqual(settings['server']['default_http_headers']['Custom-Header'], "Custom-Value") + + def test_user_settings_remove(self): + with patch.dict(settings_loader.environ, + {'SEARX_SETTINGS_PATH': join(test_dir, 'settings/user_settings_remove.yml')}): + settings, msg = settings_loader.load_settings() + self.assertTrue(msg.startswith('merge the default settings')) + self.assertEqual(settings['server']['secret_key'], "user_secret_key") + self.assertEqual(settings['server']['default_http_headers']['Custom-Header'], "Custom-Value") + engine_names = [engine['name'] for engine in settings['engines']] + self.assertNotIn('wikinews', engine_names) + self.assertNotIn('wikibooks', engine_names) + self.assertIn('wikipedia', engine_names) + + def test_user_settings_remove2(self): + with patch.dict(settings_loader.environ, + {'SEARX_SETTINGS_PATH': join(test_dir, 'settings/user_settings_remove2.yml')}): + settings, msg = settings_loader.load_settings() + self.assertTrue(msg.startswith('merge the default settings')) + self.assertEqual(settings['server']['secret_key'], "user_secret_key") + self.assertEqual(settings['server']['default_http_headers']['Custom-Header'], "Custom-Value") + engine_names = [engine['name'] for engine in settings['engines']] + self.assertNotIn('wikinews', engine_names) + self.assertNotIn('wikibooks', engine_names) + self.assertIn('wikipedia', engine_names) + wikipedia = list(filter(lambda engine: (engine.get('name')) == 'wikipedia', settings['engines'])) + self.assertEqual(wikipedia[0]['engine'], 'wikipedia') + self.assertEqual(wikipedia[0]['tokens'], ['secret_token']) + newengine = list(filter(lambda engine: (engine.get('name')) == 'newengine', settings['engines'])) + self.assertEqual(newengine[0]['engine'], 'dummy') + + def test_user_settings_keep_only(self): + with patch.dict(settings_loader.environ, + {'SEARX_SETTINGS_PATH': join(test_dir, 'settings/user_settings_keep_only.yml')}): + settings, msg = settings_loader.load_settings() + self.assertTrue(msg.startswith('merge the default settings')) + engine_names = [engine['name'] for engine in settings['engines']] + self.assertEqual(engine_names, ['wikibooks', 'wikinews', 'wikipedia', 'newengine']) + # wikipedia has been removed, then added again with the "engine" section of user_settings_keep_only.yml + self.assertEqual(len(settings['engines'][2]), 1) + + def test_custom_settings(self): + with patch.dict(settings_loader.environ, + {'SEARX_SETTINGS_PATH': join(test_dir, 'settings/user_settings.yml')}): + settings, msg = settings_loader.load_settings() + self.assertTrue(msg.startswith('load the user settings from')) + self.assertEqual(settings['server']['port'], 9000) + self.assertEqual(settings['server']['secret_key'], "user_settings_secret") + engine_names = [engine['name'] for engine in settings['engines']] + self.assertEqual(engine_names, ['wikidata', 'wikibooks', 'wikinews', 'wikiquote']) diff --git a/utils/update_user_settings.py b/utils/update_user_settings.py deleted file mode 100644 index fb6fd0b3f..000000000 --- a/utils/update_user_settings.py +++ /dev/null @@ -1,98 +0,0 @@ -#!/usr/bin/env python - -# set path -from sys import path -from os.path import realpath, dirname, join -path.append(realpath(dirname(realpath(__file__)) + '/../')) - -import argparse -import sys -import string -import ruamel.yaml -import secrets -import collections -from ruamel.yaml.scalarstring import SingleQuotedScalarString, DoubleQuotedScalarString -from searx.settings import load_settings, check_settings_yml, get_default_settings_path -from searx.exceptions import SearxSettingsException - - -RANDOM_STRING_LETTERS = string.ascii_lowercase + string.digits + string.ascii_uppercase - - -def get_random_string(): - r = [secrets.choice(RANDOM_STRING_LETTERS) for _ in range(64)] - return ''.join(r) - - -def main(prog_arg): - yaml = ruamel.yaml.YAML() - yaml.preserve_quotes = True - yaml.indent(mapping=4, sequence=1, offset=2) - user_settings_path = prog_args.get('user-settings-yaml') - - try: - default_settings, _ = load_settings(False) - if check_settings_yml(user_settings_path): - with open(user_settings_path, 'r', encoding='utf-8') as f: - user_settings = yaml.load(f.read()) - new_user_settings = False - else: - user_settings = yaml.load('use_default_settings: True') - new_user_settings = True - except SearxSettingsException as e: - sys.stderr.write(str(e)) - return - - if not new_user_settings and not user_settings.get('use_default_settings'): - sys.stderr.write('settings.yml already exists and use_default_settings is not True') - return - - user_settings['use_default_settings'] = True - use_default_settings_comment = "settings based on " + get_default_settings_path() - user_settings.yaml_add_eol_comment(use_default_settings_comment, 'use_default_settings') - - if user_settings.get('server', {}).get('secret_key') in [None, 'ultrasecretkey']: - user_settings.setdefault('server', {})['secret_key'] = DoubleQuotedScalarString(get_random_string()) - - user_engines = user_settings.get('engines') - if user_engines: - has_user_engines = True - user_engines_dict = dict((definition['name'], definition) for definition in user_engines) - else: - has_user_engines = False - user_engines_dict = {} - user_engines = [] - - # remove old engines - if prog_arg.get('add-engines') or has_user_engines: - default_engines_dict = dict((definition['name'], definition) for definition in default_settings['engines']) - for i, engine in enumerate(user_engines): - if engine['name'] not in default_engines_dict: - del user_engines[i] - - # add new engines - if prog_arg.get('add-engines'): - for engine in default_settings.get('engines', {}): - if engine['name'] not in user_engines_dict: - user_engines.append({'name': engine['name']}) - user_settings['engines'] = user_engines - - # output - if prog_arg.get('dry-run'): - yaml.dump(user_settings, sys.stdout) - else: - with open(user_settings_path, 'w', encoding='utf-8') as f: - yaml.dump(user_settings, f) - - -def parse_args(): - parser = argparse.ArgumentParser(description='Update user settings.yml') - parser.add_argument('--add-engines', dest='add-engines', default=False, action='store_true', help='Add new engines') - parser.add_argument('--dry-run', dest='dry-run', default=False, action='store_true', help='Dry run') - parser.add_argument('user-settings-yaml', type=str) - return vars(parser.parse_args()) - - -if __name__ == '__main__': - prog_args = parse_args() - main(prog_args)