Ponysearch/utils/lib_sxng_data.sh
Alexandre Flament ed66ed758d [mod] reduce memory footprint by not calling babel.Locale.parse at runtime
babel.Locale.parse loads more than 60MB in RAM.  The only purpose is to get:

    LOCALE_NAMES   - searx.data.LOCALES["LOCALE_NAMES"]
    RTL_LOCALES    - searx.data.LOCALES["RTL_LOCALES"]

This commit calls babel.Locale.parse when the translations are update from
weblate and stored in::

    searx/data/locales.json

This file can be build by::

    ./manage data.locales

By store these variables in searx.data when the translations are updated we save
round about 65MB (usually 4 worker = 260MB of RAM saved.

Suggested-by: https://github.com/searxng/searxng/discussions/2633#discussioncomment-8490494
Co-authored-by: Markus Heiser <markus.heiser@darmarit.de>
2024-02-20 10:43:20 +01:00

73 lines
2.2 KiB
Bash
Executable file

#!/usr/bin/env bash
# SPDX-License-Identifier: AGPL-3.0-or-later
data.help(){
cat <<EOF
data.:
all : update searx/sxng_locales.py and searx/data/*
traits : update searx/data/engine_traits.json & searx/sxng_locales.py
useragents: update searx/data/useragents.json with the most recent versions of Firefox
locales : update searx/data/locales.json from babel
EOF
}
data.all() {
( set -e
pyenv.activate
data.traits
data.useragents
data.locales
build_msg DATA "update searx/data/osm_keys_tags.json"
pyenv.cmd python searxng_extra/update/update_osm_keys_tags.py
build_msg DATA "update searx/data/ahmia_blacklist.txt"
python searxng_extra/update/update_ahmia_blacklist.py
build_msg DATA "update searx/data/wikidata_units.json"
python searxng_extra/update/update_wikidata_units.py
build_msg DATA "update searx/data/currencies.json"
python searxng_extra/update/update_currencies.py
build_msg DATA "update searx/data/external_bangs.json"
python searxng_extra/update/update_external_bangs.py
build_msg DATA "update searx/data/engine_descriptions.json"
python searxng_extra/update/update_engine_descriptions.py
)
}
data.traits() {
( set -e
pyenv.activate
build_msg DATA "update searx/data/engine_traits.json"
python searxng_extra/update/update_engine_traits.py
build_msg ENGINES "update searx/sxng_locales.py"
)
dump_return $?
}
data.useragents() {
build_msg DATA "update searx/data/useragents.json"
pyenv.cmd python searxng_extra/update/update_firefox_version.py
dump_return $?
}
data.locales() {
( set -e
pyenv.activate
build_msg DATA "update searx/data/locales.json"
python searxng_extra/update/update_locales.py
)
dump_return $?
}
docs.prebuild() {
build_msg DOCS "build ${DOCS_BUILD}/includes"
(
set -e
[ "$VERBOSE" = "1" ] && set -x
mkdir -p "${DOCS_BUILD}/includes"
./utils/searxng.sh searxng.doc.rst > "${DOCS_BUILD}/includes/searxng.rst"
pyenv.cmd searxng_extra/docs_prebuild
)
dump_return $?
}