Merge pull request #110 from searxng/mod-default-settings

[mod] move all default settings into searx.settings_defaults

searx/__init__.py

@@ -1,53 +1,21 @@
-'''
+# SPDX-License-Identifier: AGPL-3.0-or-later
-searx is free software: you can redistribute it and/or modify
+# lint: pylint
-it under the terms of the GNU Affero General Public License as published by
+# pylint: disable=missing-function-docstring, missing-module-docstring
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-searx is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with searx. If not, see < http://www.gnu.org/licenses/ >.
-
-(C) 2013- by Adam Tauber, <asciimoo@gmail.com>
-'''
 
+from os.path import dirname, abspath
 import logging
-import searx.settings_loader
-from os import environ
-from os.path import realpath, dirname, join, abspath, isfile
 
+import searx.settings_loader
+from searx.settings_defaults import settings_set_defaults
 
 searx_dir = abspath(dirname(__file__))
 searx_parent_dir = abspath(dirname(dirname(__file__)))
-engine_dir = dirname(realpath(__file__))
-static_path = abspath(join(dirname(__file__), 'static'))
 settings, settings_load_message = searx.settings_loader.load_settings()
 
-if settings['ui']['static_path']:
+if settings is not None:
-    static_path = settings['ui']['static_path']
+    settings = settings_set_defaults(settings)
-
-'''
-enable debug if
-the environnement variable SEARX_DEBUG is 1 or true
-(whatever the value in settings.yml)
-or general.debug=True in settings.yml
-disable debug if
-the environnement variable SEARX_DEBUG is 0 or false
-(whatever the value in settings.yml)
-or general.debug=False in settings.yml
-'''
-searx_debug_env = environ.get('SEARX_DEBUG', '').lower()
-if searx_debug_env == 'true' or searx_debug_env == '1':
-    searx_debug = True
-elif searx_debug_env == 'false' or searx_debug_env == '0':
-    searx_debug = False
-else:
-    searx_debug = settings.get('general', {}).get('debug')
 
+searx_debug = settings['general']['debug']
 if searx_debug:
     logging.basicConfig(level=logging.DEBUG)
 else:
@@ -55,15 +23,16 @@ else:
 
 logger = logging.getLogger('searx')
 logger.info(settings_load_message)
-logger.info('Initialisation done')
 
-if 'SEARX_SECRET' in environ:
+# log max_request_timeout
-    settings['server']['secret_key'] = environ['SEARX_SECRET']
+max_request_timeout = settings['outgoing']['max_request_timeout']
-if 'SEARX_BIND_ADDRESS' in environ:
+if max_request_timeout is None:
-    settings['server']['bind_address'] = environ['SEARX_BIND_ADDRESS']
+    logger.info('max_request_timeout=%s', repr(max_request_timeout))
+else:
+    logger.info('max_request_timeout=%i second(s)', max_request_timeout)
 
 
-class _brand_namespace:
+class _brand_namespace:  # pylint: disable=invalid-name
 
     @classmethod
     def get_val(cls, group, name, default=''):

searx/engines/__init__.py

@@ -144,7 +144,7 @@ def load_engine(engine_data):
         # exclude onion engines if not using tor.
         return None
 
-    engine.timeout += settings['outgoing'].get('extra_proxy_timeout', 0)
+    engine.timeout += settings['outgoing']['extra_proxy_timeout']
 
     for category_name in engine.categories:
         categories.setdefault(category_name, []).append(engine)

searx/network/network.py

@@ -224,28 +224,22 @@ def initialize(settings_engines=None, settings_outgoing=None):
 
     global NETWORKS
 
-    settings_engines = settings_engines or settings.get('engines')
+    settings_engines = settings_engines or settings['engines']
-    settings_outgoing = settings_outgoing or settings.get('outgoing')
+    settings_outgoing = settings_outgoing or settings['outgoing']
 
     # default parameters for AsyncHTTPTransport
     # see https://github.com/encode/httpx/blob/e05a5372eb6172287458b37447c30f650047e1b8/httpx/_transports/default.py#L108-L121  # pylint: disable=line-too-long
     default_params = {
         'enable_http': False,
         'verify': True,
-        'enable_http2': settings_outgoing.get('enable_http2', True),
+        'enable_http2': settings_outgoing['enable_http2'],
-        # Magic number kept from previous code
+        'max_connections': settings_outgoing['pool_connections'],
-        'max_connections': settings_outgoing.get('pool_connections', 100),
+        'max_keepalive_connections': settings_outgoing['pool_maxsize'],
-        # Picked from constructor
+        'keepalive_expiry': settings_outgoing['keepalive_expiry'],
-        'max_keepalive_connections': settings_outgoing.get('pool_maxsize', 10),
+        'local_addresses': settings_outgoing['source_ips'],
-        #
+        'proxies': settings_outgoing['proxies'],
-        'keepalive_expiry': settings_outgoing.get('keepalive_expiry', 5.0),
+        'max_redirects': settings_outgoing['max_redirects'],
-        'local_addresses': settings_outgoing.get('source_ips'),
+        'retries': settings_outgoing['retries'],
-        'proxies': settings_outgoing.get('proxies'),
-        # default maximum redirect
-        # from https://github.com/psf/requests/blob/8c211a96cdbe9fe320d63d9e1ae15c5c07e179f8/requests/models.py#L55
-        'max_redirects': settings_outgoing.get('max_redirects', 30),
-        #
-        'retries': settings_outgoing.get('retries', 0),
         'retry_on_http_error': None,
     }
 
@@ -274,7 +268,7 @@ def initialize(settings_engines=None, settings_outgoing=None):
     NETWORKS['ipv6'] = new_network({'local_addresses': '::'})
 
     # define networks from outgoing.networks
-    for network_name, network in settings_outgoing.get('networks', {}).items():
+    for network_name, network in settings_outgoing['networks'].items():
         NETWORKS[network_name] = new_network(network)
 
     # define networks from engines.[i].network (except references)

searx/plugins/__init__.py

@@ -21,7 +21,7 @@ from os import listdir, makedirs, remove, stat, utime
 from os.path import abspath, basename, dirname, exists, join
 from shutil import copyfile
 
-from searx import logger, settings, static_path
+from searx import logger, settings
 
 
 logger = logger.getChild('plugins')
@@ -123,7 +123,7 @@ def sync_resource(base_path, resource_path, name, target_dir, plugin_dir):
 
 def prepare_package_resources(pkg, name):
     plugin_dir = 'plugin_' + name
-    target_dir = join(static_path, 'plugins/external_plugins', plugin_dir)
+    target_dir = join(settings['ui']['static_path'], 'plugins/external_plugins', plugin_dir)
     try:
         makedirs(target_dir, exist_ok=True)
     except:
@@ -170,10 +170,10 @@ plugins.register(search_on_category_select)
 plugins.register(tracker_url_remover)
 plugins.register(vim_hotkeys)
 # load external plugins
-if 'plugins' in settings:
+if settings['plugins']:
     plugins.register(*settings['plugins'], external=True)
 
-if 'enabled_plugins' in settings:
+if settings['enabled_plugins']:
     for plugin in plugins:
         if plugin.name in settings['enabled_plugins']:
             plugin.default_on = True
@@ -181,5 +181,5 @@ if 'enabled_plugins' in settings:
             plugin.default_on = False
 
 # load tor specific plugins
-if settings['outgoing'].get('using_tor_proxy'):
+if settings['outgoing']['using_tor_proxy']:
     plugins.register(ahmia_filter)

searx/preferences.py

@@ -333,25 +333,25 @@ class Preferences:
                 choices=categories + ['none']
             ),
             'language': SearchLanguageSetting(
-                settings['search'].get('default_lang', ''),
+                settings['search']['default_lang'],
                 is_locked('language'),
                 choices=list(LANGUAGE_CODES) + ['']
             ),
             'locale': EnumStringSetting(
-                settings['ui'].get('default_locale', ''),
+                settings['ui']['default_locale'],
                 is_locked('locale'),
                 choices=list(settings['locales'].keys()) + ['']
             ),
             'autocomplete': EnumStringSetting(
-                settings['search'].get('autocomplete', ''),
+                settings['search']['autocomplete'],
                 is_locked('autocomplete'),
                 choices=list(autocomplete.backends.keys()) + ['']
             ),
             'image_proxy': MapSetting(
-                settings['server'].get('image_proxy', False),
+                settings['server']['image_proxy'],
                 is_locked('image_proxy'),
                 map={
-                    '': settings['server'].get('image_proxy', 0),
+                    '': settings['server']['image_proxy'],
                     '0': False,
                     '1': True,
                     'True': True,
@@ -359,12 +359,12 @@ class Preferences:
                 }
             ),
             'method': EnumStringSetting(
-                settings['server'].get('method', 'POST'),
+                settings['server']['method'],
                 is_locked('method'),
                 choices=('GET', 'POST')
             ),
             'safesearch': MapSetting(
-                settings['search'].get('safe_search', 0),
+                settings['search']['safe_search'],
                 is_locked('safesearch'),
                 map={
                     '0': 0,
@@ -373,12 +373,12 @@ class Preferences:
                 }
             ),
             'theme': EnumStringSetting(
-                settings['ui'].get('default_theme', 'oscar'),
+                settings['ui']['default_theme'],
                 is_locked('theme'),
                 choices=themes
             ),
             'results_on_new_tab': MapSetting(
-                settings['ui'].get('results_on_new_tab', False),
+                settings['ui']['results_on_new_tab'],
                 is_locked('results_on_new_tab'),
                 map={
                     '0': False,
@@ -393,11 +393,11 @@ class Preferences:
                 choices=DOI_RESOLVERS
             ),
             'oscar-style': EnumStringSetting(
-                settings['ui'].get('theme_args', {}).get('oscar_style', 'logicodev'),
+                settings['ui']['theme_args']['oscar_style'],
                 is_locked('oscar-style'),
                 choices=['', 'logicodev', 'logicodev-dark', 'pointhi']),
             'advanced_search': MapSetting(
-                settings['ui'].get('advanced_search', False),
+                settings['ui']['advanced_search'],
                 is_locked('advanced_search'),
                 map={
                     '0': False,

searx/search/__init__.py

@@ -23,17 +23,6 @@ from searx.search.checker import initialize as initialize_checker
 
 logger = logger.getChild('search')
 
-max_request_timeout = settings.get('outgoing', {}).get('max_request_timeout' or None)
-if max_request_timeout is None:
-    logger.info('max_request_timeout={0}'.format(max_request_timeout))
-else:
-    if isinstance(max_request_timeout, float):
-        logger.info('max_request_timeout={0} second(s)'.format(max_request_timeout))
-    else:
-        logger.critical('outgoing.max_request_timeout if defined has to be float')
-        import sys
-        sys.exit(1)
-
 
 def initialize(settings_engines=None, enable_checker=False):
     settings_engines = settings_engines or settings['engines']
@@ -115,6 +104,7 @@ class Search:
             default_timeout = max(default_timeout, processor.engine.timeout)
 
         # adjust timeout
+        max_request_timeout = settings['outgoing']['max_request_timeout']
         actual_timeout = default_timeout
         query_timeout = self.search_query.timeout_limit
 

searx/settings_defaults.py

@@ -0,0 +1,202 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+# lint: pylint
+# pylint: disable=missing-function-docstring
+"""Implementation of the default settings.
+
+"""
+
+import typing
+import numbers
+import errno
+import os
+import logging
+from os.path import dirname, abspath
+
+from searx.languages import language_codes as languages
+
+searx_dir = abspath(dirname(__file__))
+
+logger = logging.getLogger('searx')
+OUTPUT_FORMATS = ['html', 'csv', 'json', 'rss']
+LANGUAGE_CODES = ('', 'all') + tuple(l[0] for l in languages)
+OSCAR_STYLE = ('logicodev', 'logicodev-dark', 'pointhi')
+CATEGORY_ORDER = [
+    'general',
+    'images',
+    'videos',
+    'news',
+    'map',
+    'music',
+    'it',
+    'science',
+    'files',
+    'social medias',
+]
+STR_TO_BOOL = {
+    '0': False,
+    'false': False,
+    'off': False,
+    '1': True,
+    'true': True,
+    'on': True,
+}
+_UNDEFINED = object()
+
+
+class SettingsValue:
+    """Check and update a setting value
+    """
+
+    def __init__(self,
+                 type_definition: typing.Union[None, typing.Any, typing.Tuple[typing.Any]]=None,
+                 default: typing.Any=None,
+                 environ_name: str=None):
+        self.type_definition = (
+            type_definition
+            if type_definition is None or isinstance(type_definition, tuple)
+            else (type_definition,)
+        )
+        self.default = default
+        self.environ_name = environ_name
+
+    @property
+    def type_definition_repr(self):
+        types_str = [
+            t.__name__ if isinstance(t, type) else repr(t)
+            for t in self.type_definition
+        ]
+        return ', '.join(types_str)
+
+    def check_type_definition(self, value: typing.Any) -> None:
+        if value in self.type_definition:
+            return
+        type_list = tuple(t for t in self.type_definition if isinstance(t, type))
+        if not isinstance(value, type_list):
+            raise ValueError(
+                'The value has to be one of these types/values: {}'.format(
+                    self.type_definition_repr))
+
+    def __call__(self, value: typing.Any) -> typing.Any:
+        if value == _UNDEFINED:
+            value = self.default
+        # override existing value with environ
+        if self.environ_name and self.environ_name in os.environ:
+            value = os.environ[self.environ_name]
+            if self.type_definition == (bool,):
+                value = STR_TO_BOOL[value.lower()]
+
+        self.check_type_definition(value)
+        return value
+
+
+class SettingsDirectoryValue(SettingsValue):
+    """Check and update a setting value that is a directory path
+    """
+
+    def check_type_definition(self, value: typing.Any) -> typing.Any:
+        super().check_type_definition(value)
+        if not os.path.isdir(value):
+            raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), value)
+
+    def __call__(self, value: typing.Any) -> typing.Any:
+        if value == '':
+            value = self.default
+        return super().__call__(value)
+
+
+def apply_schema(settings, schema, path_list):
+    error = False
+    for key, value in schema.items():
+        if isinstance(value, SettingsValue):
+            try:
+                settings[key] = value(settings.get(key, _UNDEFINED))
+            except Exception as e:  # pylint: disable=broad-except
+                # don't stop now: check other values
+                logger.error('%s: %s', '.'.join([*path_list, key]), e)
+                error = True
+        elif isinstance(value, dict):
+            error = error or apply_schema(settings.setdefault(key, {}), schema[key], [*path_list, key])
+        else:
+            settings.setdefault(key, value)
+    if len(path_list) == 0 and error:
+        raise ValueError('Invalid settings.yml')
+    return error
+
+
+SCHEMA = {
+    'general': {
+        'debug': SettingsValue(bool, False, 'SEARX_DEBUG'),
+        'instance_name': SettingsValue(str, 'searxng'),
+        'contact_url': SettingsValue((None, False, str), None),
+    },
+    'brand': {
+    },
+    'search': {
+        'safe_search': SettingsValue((0,1,2), 0),
+        'autocomplete': SettingsValue(str, ''),
+        'default_lang': SettingsValue(LANGUAGE_CODES, ''),
+        'ban_time_on_fail': SettingsValue(numbers.Real, 5),
+        'max_ban_time_on_fail': SettingsValue(numbers.Real, 120),
+        'formats': SettingsValue(list, OUTPUT_FORMATS),
+    },
+    'server': {
+        'port': SettingsValue(int, 8888),
+        'bind_address': SettingsValue(str, '127.0.0.1', 'SEARX_BIND_ADDRESS'),
+        'secret_key': SettingsValue(str, environ_name='SEARX_SECRET'),
+        'base_url': SettingsValue((False, str), False),
+        'image_proxy': SettingsValue(bool, False),
+        'http_protocol_version': SettingsValue(('1.0', '1.1'), '1.0'),
+        'method': SettingsValue(('POST', 'GET'), 'POST'),
+        'default_http_headers': SettingsValue(dict, {}),
+    },
+    'ui': {
+        'static_path': SettingsDirectoryValue(str, os.path.join(searx_dir, 'static')),
+        'templates_path': SettingsDirectoryValue(str, os.path.join(searx_dir, 'templates')),
+        'default_theme': SettingsValue(str, 'oscar'),
+        'default_locale': SettingsValue(str, ''),
+        'theme_args': {
+            'oscar_style': SettingsValue(OSCAR_STYLE, 'logicodev'),
+        },
+        'results_on_new_tab': SettingsValue(bool, False),
+        'advanced_search': SettingsValue(bool, False),
+        'categories_order': SettingsValue(list, CATEGORY_ORDER),
+    },
+    'preferences': {
+        'lock': SettingsValue(list, []),
+    },
+    'outgoing': {
+        'useragent_suffix': SettingsValue(str, ''),
+        'request_timeout': SettingsValue(numbers.Real, 3.0),
+        'enable_http2': SettingsValue(bool, True),
+        'max_request_timeout': SettingsValue((None, numbers.Real), None),
+        # Magic number kept from previous code
+        'pool_connections': SettingsValue(int, 100),
+        # Picked from constructor
+        'pool_maxsize': SettingsValue(int, 10),
+        'keepalive_expiry': SettingsValue(numbers.Real, 5.0),
+        # default maximum redirect
+        # from https://github.com/psf/requests/blob/8c211a96cdbe9fe320d63d9e1ae15c5c07e179f8/requests/models.py#L55
+        'max_redirects': SettingsValue(int, 30),
+        'retries': SettingsValue(int, 0),
+        'proxies': SettingsValue((None, str, dict), None),
+        'source_ips': SettingsValue((None, str, list), None),
+        # Tor configuration
+        'using_tor_proxy': SettingsValue(bool, False),
+        'extra_proxy_timeout': SettingsValue(int, 0),
+        'networks': {
+        },
+    },
+    'plugins': SettingsValue((None, list), None),
+    'enabled_plugins': SettingsValue(list, []),
+    'checker': {
+        'off_when_debug': SettingsValue(bool, True),
+    },
+    'engines': SettingsValue(list, []),
+    'locales': SettingsValue(dict, {'en': 'English'}),
+    'doi_resolvers': {
+    },
+}
+
+def settings_set_defaults(settings):
+    apply_schema(settings, SCHEMA, [])
+    return settings

searx/utils.py

@@ -8,7 +8,6 @@ from os.path import splitext, join
 from random import choice
 from html.parser import HTMLParser
 from urllib.parse import urljoin, urlparse
-from collections.abc import Mapping
 
 from lxml import html
 from lxml.etree import ElementBase, XPath, XPathError, XPathSyntaxError, _ElementStringResult, _ElementUnicodeResult
@@ -46,7 +45,7 @@ def searx_useragent():
     """Return the searx User Agent"""
     return 'searx/{searx_version} {suffix}'.format(
            searx_version=VERSION_STRING,
-           suffix=settings['outgoing'].get('useragent_suffix', '')).strip()
+           suffix=settings['outgoing']['useragent_suffix'].strip())
 
 
 def gen_useragent(os=None):
@@ -501,58 +500,6 @@ def get_engine_from_settings(name):
     return {}
 
 
-NOT_EXISTS = object()
-"""Singleton used by :py:obj:`get_value` if a key does not exists."""
-
-
-def get_value(dictionary, *keys, default=NOT_EXISTS):
-    """Return the value from a *deep* mapping type (e.g. the ``settings`` object
-    from yaml).  If the path to the *key* does not exists a :py:obj:`NOT_EXISTS`
-    is returned (non ``KeyError`` exception is raised).
-
-    .. code: python
-
-       >>> from searx import settings
-       >>> from searx.utils import get_value, NOT_EXISTS
-       >>> get_value(settings, 'checker', 'additional_tests', 'rosebud', 'result_container')
-       ['not_empty', ['one_title_contains', 'citizen kane']]
-
-       >>> get_value(settings, 'search', 'xxx') is NOT_EXISTS
-       True
-       >>> get_value(settings, 'search', 'formats')
-       ['html', 'csv', 'json', 'rss']
-
-    The list returned from the ``search.format`` key is not a mapping type, you
-    can't traverse along non-mapping types.  If you try it, you will get a
-    :py:ref:`NOT_EXISTS`:
-
-    .. code: python
-
-       >>> get_value(settings, 'search', 'format', 'csv') is NOT_EXISTS
-       True
-       >>> get_value(settings, 'search', 'formats')[1]
-       'csv'
-
-    For convenience you can replace :py:ref:`NOT_EXISTS` by a default value of
-    your choice:
-
-    .. code: python
-
-       if 'csv' in get_value(settings, 'search', 'formats', default=[]):
-           print("csv format is denied")
-
-    """
-
-    obj = dictionary
-    for k in keys:
-        if not isinstance(obj, Mapping):
-            raise TypeError("expected mapping type, got %s" % type(obj))
-        obj = obj.get(k, default)
-        if obj is default:
-            return obj
-    return obj
-
-
 def get_xpath(xpath_spec):
     """Return cached compiled XPath
 

searx/webapp.py

@@ -56,12 +56,12 @@ from flask_babel import (
 )
 
 from searx import logger
-from searx import brand, static_path
+from searx import brand
 from searx import (
     settings,
-    searx_dir,
     searx_debug,
 )
+from searx.settings_defaults import OUTPUT_FORMATS
 from searx.exceptions import SearxParameterException
 from searx.engines import (
     categories,
@@ -71,7 +71,6 @@ from searx.engines import (
 from searx.webutils import (
     UnicodeWriter,
     highlight_content,
-    get_resources_directory,
     get_static_files,
     get_result_templates,
     get_themes,
@@ -88,7 +87,6 @@ from searx.utils import (
     gen_useragent,
     dict_subset,
     match_language,
-    get_value,
 )
 from searx.version import VERSION_STRING
 from searx.query import RawTextQuery
@@ -139,7 +137,7 @@ if sys.version_info[0] < 3:
 logger = logger.getChild('webapp')
 
 # serve pages with HTTP/1.1
-WSGIRequestHandler.protocol_version = "HTTP/{}".format(settings['server'].get('http_protocol_version', '1.0'))
+WSGIRequestHandler.protocol_version = "HTTP/{}".format(settings['server']['http_protocol_version'])
 
 # check secret_key
 if not searx_debug and settings['server']['secret_key'] == 'ultrasecretkey':
@@ -147,25 +145,22 @@ if not searx_debug and settings['server']['secret_key'] == 'ultrasecretkey':
     sys.exit(1)
 
 # about static
-static_path = get_resources_directory(searx_dir, 'static', settings['ui']['static_path'])
+logger.debug('static directory is %s', settings['ui']['static_path'])
-logger.debug('static directory is %s', static_path)
+static_files = get_static_files(settings['ui']['static_path'])
-static_files = get_static_files(static_path)
 
 # about templates
+logger.debug('templates directory is %s', settings['ui']['templates_path'])
 default_theme = settings['ui']['default_theme']
-templates_path = get_resources_directory(searx_dir, 'templates', settings['ui']['templates_path'])
+templates_path = settings['ui']['templates_path']
-logger.debug('templates directory is %s', templates_path)
 themes = get_themes(templates_path)
 result_templates = get_result_templates(templates_path)
 global_favicons = []
 for indice, theme in enumerate(themes):
     global_favicons.append([])
-    theme_img_path = os.path.join(static_path, 'themes', theme, 'img', 'icons')
+    theme_img_path = os.path.join(settings['ui']['static_path'], 'themes', theme, 'img', 'icons')
     for (dirpath, dirnames, filenames) in os.walk(theme_img_path):
         global_favicons[indice].extend(filenames)
 
-OUTPUT_FORMATS = ['html', 'csv', 'json', 'rss']
-
 STATS_SORT_PARAMETERS = {
     'name': (False, 'name', ''),
     'score': (True, 'score', 0),
@@ -177,7 +172,7 @@ STATS_SORT_PARAMETERS = {
 # Flask app
 app = Flask(
     __name__,
-    static_folder=static_path,
+    static_folder=settings['ui']['static_path'],
     template_folder=templates_path
 )
 
@@ -517,8 +512,7 @@ def render(template_name, override_theme=None, **kwargs):
     kwargs['preferences'] = request.preferences
 
     kwargs['search_formats'] = [
-        x for x in get_value(
+        x for x in settings['search']['formats']
-            settings, 'search', 'formats', default=OUTPUT_FORMATS)
         if x != 'html']
 
     kwargs['brand'] = brand
@@ -545,12 +539,7 @@ def render(template_name, override_theme=None, **kwargs):
 
 
 def _get_ordered_categories():
-    ordered_categories = []
+    ordered_categories = list(settings['ui']['categories_order'])
-    if 'categories_order' not in settings['ui']:
-        ordered_categories = ['general']
-        ordered_categories.extend(x for x in sorted(categories.keys()) if x != 'general')
-        return ordered_categories
-    ordered_categories = settings['ui']['categories_order']
     ordered_categories.extend(x for x in sorted(categories.keys()) if x not in ordered_categories)
     return ordered_categories
 
@@ -610,7 +599,7 @@ def pre_request():
 @app.after_request
 def add_default_headers(response):
     # set default http headers
-    for header, value in settings['server'].get('default_http_headers', {}).items():
+    for header, value in settings['server']['default_http_headers'].items():
         if header in response.headers:
             continue
         response.headers[header] = value
@@ -696,7 +685,7 @@ def search():
     if output_format not in OUTPUT_FORMATS:
         output_format = 'html'
 
-    if output_format not in get_value(settings, 'search', 'formats', default=OUTPUT_FORMATS):
+    if output_format not in settings['search']['formats']:
         flask.abort(403)
 
     # check if there is query (not None and not an empty string)
@@ -1069,11 +1058,6 @@ def preferences():
             'time_range_support': time_range_support,
         }
 
-    #
-    locked_preferences = list()
-    if 'preferences' in settings and 'lock' in settings['preferences']:
-        locked_preferences = settings['preferences']['lock']
-
     #
     return render('preferences.html',
                   selected_categories=get_selected_categories(request.preferences, request.form),
@@ -1098,7 +1082,7 @@ def preferences():
                   theme=get_current_theme_name(),
                   preferences_url_params=request.preferences.get_as_url_params(),
                   base_url=get_base_url(),
-                  locked_preferences=locked_preferences,
+                  locked_preferences=settings['preferences']['lock'],
                   preferences=True)
 
 
@@ -1271,7 +1255,7 @@ def favicon():
     return send_from_directory(
         os.path.join(
             app.root_path,
-            static_path,
+            settings['ui']['static_path'],
             'themes',
             get_current_theme_name(),
             'img'),

searx/webutils.py

@@ -47,14 +47,6 @@ class UnicodeWriter:
             self.writerow(row)
 
 
-def get_resources_directory(searx_directory, subdirectory, resources_directory):
-    if not resources_directory:
-        resources_directory = os.path.join(searx_directory, subdirectory)
-    if not os.path.isdir(resources_directory):
-        raise Exception(resources_directory + " is not a directory")
-    return resources_directory
-
-
 def get_themes(templates_path):
     """Returns available themes list."""
     themes = os.listdir(templates_path)

tests/unit/test_search.py

@@ -2,6 +2,7 @@
 
 from searx.testing import SearxTestCase
 from searx.search import SearchQuery, EngineRef
+from searx import settings
 import searx.search
 
 
@@ -41,7 +42,7 @@ class SearchTestCase(SearxTestCase):
         searx.search.initialize(TEST_ENGINES)
 
     def test_timeout_simple(self):
-        searx.search.max_request_timeout = None
+        settings['outgoing']['max_request_timeout'] = None
         search_query = SearchQuery('test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')],
                                    'en-US', SAFESEARCH, PAGENO, None, None)
         search = searx.search.Search(search_query)
@@ -49,7 +50,7 @@ class SearchTestCase(SearxTestCase):
         self.assertEqual(search.actual_timeout, 3.0)
 
     def test_timeout_query_above_default_nomax(self):
-        searx.search.max_request_timeout = None
+        settings['outgoing']['max_request_timeout'] = None
         search_query = SearchQuery('test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')],
                                    'en-US', SAFESEARCH, PAGENO, None, 5.0)
         search = searx.search.Search(search_query)
@@ -57,7 +58,7 @@ class SearchTestCase(SearxTestCase):
         self.assertEqual(search.actual_timeout, 3.0)
 
     def test_timeout_query_below_default_nomax(self):
-        searx.search.max_request_timeout = None
+        settings['outgoing']['max_request_timeout'] = None
         search_query = SearchQuery('test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')],
                                    'en-US', SAFESEARCH, PAGENO, None, 1.0)
         search = searx.search.Search(search_query)
@@ -65,7 +66,7 @@ class SearchTestCase(SearxTestCase):
         self.assertEqual(search.actual_timeout, 1.0)
 
     def test_timeout_query_below_max(self):
-        searx.search.max_request_timeout = 10.0
+        settings['outgoing']['max_request_timeout'] = 10.0
         search_query = SearchQuery('test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')],
                                    'en-US', SAFESEARCH, PAGENO, None, 5.0)
         search = searx.search.Search(search_query)
@@ -73,7 +74,7 @@ class SearchTestCase(SearxTestCase):
         self.assertEqual(search.actual_timeout, 5.0)
 
     def test_timeout_query_above_max(self):
-        searx.search.max_request_timeout = 10.0
+        settings['outgoing']['max_request_timeout'] = 10.0
         search_query = SearchQuery('test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')],
                                    'en-US', SAFESEARCH, PAGENO, None, 15.0)
         search = searx.search.Search(search_query)