forked from Ponysearch/Ponysearch
[doc] improve documentation of make targets and ./manage script
BTW force modularization of the ./mange script into sub modules: - utils/lib_sxng_data.sh - utils/lib_sxng_node.sh - utils/lib_sxng_static.sh - utils/lib_sxng_test.sh - utils/lib_sxng_themes.sh Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
This commit is contained in:
Markus Heiser
Markus Heiser
parent
1d0abb7157
commit
64100db904
16 changed files with 767 additions and 654 deletions
12
Makefile
12
Makefile
|
|
@ -59,16 +59,16 @@ test.shell:
|
||||||
utils/brand.env \
|
utils/brand.env \
|
||||||
$(MTOOLS) \
|
$(MTOOLS) \
|
||||||
utils/lib.sh \
|
utils/lib.sh \
|
||||||
utils/lib_nvm.sh \
|
utils/lib_sxng*.sh \
|
||||||
utils/lib_static.sh \
|
|
||||||
utils/lib_go.sh \
|
utils/lib_go.sh \
|
||||||
|
utils/lib_nvm.sh \
|
||||||
utils/lib_redis.sh \
|
utils/lib_redis.sh \
|
||||||
utils/filtron.sh \
|
|
||||||
utils/searx.sh \
|
|
||||||
utils/searxng.sh \
|
utils/searxng.sh \
|
||||||
utils/morty.sh \
|
|
||||||
utils/lxc.sh \
|
utils/lxc.sh \
|
||||||
utils/lxc-searxng.env
|
utils/lxc-searxng.env \
|
||||||
|
utils/searx.sh \
|
||||||
|
utils/filtron.sh \
|
||||||
|
utils/morty.sh
|
||||||
$(Q)$(MTOOLS) build_msg TEST "$@ OK"
|
$(Q)$(MTOOLS) build_msg TEST "$@ OK"
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -4,22 +4,30 @@
|
||||||
Buildhosts
|
Buildhosts
|
||||||
==========
|
==========
|
||||||
|
|
||||||
.. sidebar:: This article needs some work
|
|
||||||
|
|
||||||
If you have any contribution send us your :pull:`PR <../pulls>`, see
|
|
||||||
:ref:`how to contribute`.
|
|
||||||
|
|
||||||
.. contents::
|
.. contents::
|
||||||
:depth: 2
|
:depth: 2
|
||||||
:local:
|
:local:
|
||||||
:backlinks: entry
|
:backlinks: entry
|
||||||
|
|
||||||
To get best results from build, it's recommend to install additional packages
|
To get best results from build, it's recommend to install additional packages on
|
||||||
on build hosts (see :ref:`searxng.sh`).::
|
build hosts (see :ref:`searxng.sh`).
|
||||||
|
|
||||||
sudo -H ./utils/searxng.sh install buildhost
|
.. _searxng.sh install buildhost:
|
||||||
|
|
||||||
This will install packages needed by searx:
|
Build and Development tools
|
||||||
|
===========================
|
||||||
|
|
||||||
|
To Install tools used by build and development tasks in once:
|
||||||
|
|
||||||
|
.. tabs::
|
||||||
|
|
||||||
|
.. group-tab:: SearXNG's development tools
|
||||||
|
|
||||||
|
.. code:: sh
|
||||||
|
|
||||||
|
$ sudo -H ./utils/searxng.sh install buildhost
|
||||||
|
|
||||||
|
This will install packages needed by SearXNG:
|
||||||
|
|
||||||
.. kernel-include:: $DOCS_BUILD/includes/searxng.rst
|
.. kernel-include:: $DOCS_BUILD/includes/searxng.rst
|
||||||
:start-after: START distro-packages
|
:start-after: START distro-packages
|
||||||
|
|
@ -73,7 +81,7 @@ If your docs build (``make docs.html``) shows warnings like this::
|
||||||
display), check the imgmath_latex setting
|
display), check the imgmath_latex setting
|
||||||
|
|
||||||
you need to install additional packages on your build host, to get better HTML
|
you need to install additional packages on your build host, to get better HTML
|
||||||
output.
|
output (:ref:`install buildhost <searxng.sh install buildhost>`).
|
||||||
|
|
||||||
.. tabs::
|
.. tabs::
|
||||||
|
|
||||||
|
|
@ -93,7 +101,7 @@ output.
|
||||||
|
|
||||||
.. code-block:: sh
|
.. code-block:: sh
|
||||||
|
|
||||||
$ sudo dnf install graphviz graphviz-gd texlive-xetex-bin librsvg2-tools
|
$ sudo dnf install graphviz graphviz-gd ImageMagick texlive-xetex-bin librsvg2-tools
|
||||||
|
|
||||||
|
|
||||||
For PDF output you also need:
|
For PDF output you also need:
|
||||||
|
|
@ -118,8 +126,7 @@ For PDF output you also need:
|
||||||
|
|
||||||
$ sudo dnf install \
|
$ sudo dnf install \
|
||||||
texlive-collection-fontsrecommended texlive-collection-latex \
|
texlive-collection-fontsrecommended texlive-collection-latex \
|
||||||
dejavu-sans-fonts dejavu-serif-fonts dejavu-sans-mono-fonts \
|
dejavu-sans-fonts dejavu-serif-fonts dejavu-sans-mono-fonts
|
||||||
ImageMagick
|
|
||||||
|
|
||||||
.. _sh lint:
|
.. _sh lint:
|
||||||
|
|
||||||
|
|
@ -128,7 +135,8 @@ Lint shell scripts
|
||||||
|
|
||||||
.. _ShellCheck: https://github.com/koalaman/shellcheck
|
.. _ShellCheck: https://github.com/koalaman/shellcheck
|
||||||
|
|
||||||
To lint shell scripts, we use ShellCheck_ - a shell script static analysis tool.
|
To lint shell scripts we use ShellCheck_ - a shell script static analysis tool
|
||||||
|
(:ref:`install buildhost <searxng.sh install buildhost>`).
|
||||||
|
|
||||||
.. SNIP sh lint requirements
|
.. SNIP sh lint requirements
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -24,7 +24,10 @@ developer) account needs to be added to the *searxng-redis* group.
|
||||||
rediss://[[username]:[password]]@localhost:6379/0
|
rediss://[[username]:[password]]@localhost:6379/0
|
||||||
unix://[[username]:[password]]@/path/to/socket.sock?db=0
|
unix://[[username]:[password]]@/path/to/socket.sock?db=0
|
||||||
|
|
||||||
.. admonition:: Tip for developers
|
.. _Redis Developer Notes:
|
||||||
|
|
||||||
|
Redis Developer Notes
|
||||||
|
=====================
|
||||||
|
|
||||||
To set up a local redis instance, first set the socket path of the Redis DB
|
To set up a local redis instance, first set the socket path of the Redis DB
|
||||||
in your YAML setting:
|
in your YAML setting:
|
||||||
|
|
@ -34,7 +37,10 @@ developer) account needs to be added to the *searxng-redis* group.
|
||||||
redis:
|
redis:
|
||||||
url: unix:///usr/local/searxng-redis/run/redis.sock?db=0
|
url: unix:///usr/local/searxng-redis/run/redis.sock?db=0
|
||||||
|
|
||||||
Then use the following commands to install the redis instance ::
|
Then use the following commands to install the redis instance (:ref:`manage
|
||||||
|
redis.help`):
|
||||||
|
|
||||||
|
.. code:: sh
|
||||||
|
|
||||||
$ ./manage redis.build
|
$ ./manage redis.build
|
||||||
$ sudo -H ./manage redis.install
|
$ sudo -H ./manage redis.install
|
||||||
|
|
|
||||||
|
|
@ -1,53 +0,0 @@
|
||||||
.. START create user
|
|
||||||
|
|
||||||
.. tabs::
|
|
||||||
|
|
||||||
.. group-tab:: bash
|
|
||||||
|
|
||||||
.. code-block:: sh
|
|
||||||
|
|
||||||
$ sudo -H useradd --shell /bin/bash --system \\
|
|
||||||
--home-dir \"$SERVICE_HOME\" \\
|
|
||||||
--comment \"Privacy-respecting metasearch engine\" $SERVICE_USER
|
|
||||||
|
|
||||||
$ sudo -H mkdir \"$SERVICE_HOME\"
|
|
||||||
$ sudo -H chown -R \"$SERVICE_GROUP:$SERVICE_GROUP\" \"$SERVICE_HOME\"
|
|
||||||
|
|
||||||
.. END create user
|
|
||||||
|
|
||||||
.. START install go
|
|
||||||
|
|
||||||
.. tabs::
|
|
||||||
|
|
||||||
.. group-tab:: os: linux / arch: amd64
|
|
||||||
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
$ cat > \"$GO_ENV\" <<EOF
|
|
||||||
export GOPATH=${SERVICE_HOME}/go-apps
|
|
||||||
export PATH=\$PATH:${SERVICE_HOME}/local/go/bin:\$GOPATH/bin
|
|
||||||
EOF
|
|
||||||
$ sudo -i -u \"${SERVICE_USER}\"
|
|
||||||
(${SERVICE_USER}) $ echo 'source $GO_ENV' >> ~/.profile
|
|
||||||
(${SERVICE_USER}) $ mkdir ${SERVICE_HOME}/local
|
|
||||||
(${SERVICE_USER}) $ wget --progress=bar -O \"${GO_VERSION}.linux-amd64.tar.gz\" \\
|
|
||||||
\"${GO_DL_URL}/${GO_VERSION}.linux-amd64.tar.gz\"
|
|
||||||
(${SERVICE_USER}) $ tar -C ${SERVICE_HOME}/local -xzf \"${GO_VERSION}.linux-amd64.tar.gz\"
|
|
||||||
(${SERVICE_USER}) $ which go
|
|
||||||
${SERVICE_HOME}/local/go/bin/go
|
|
||||||
|
|
||||||
.. END install go
|
|
||||||
|
|
||||||
.. START install filtron
|
|
||||||
|
|
||||||
.. tabs::
|
|
||||||
|
|
||||||
.. group-tab:: bash
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
$ sudo -i -u \"${SERVICE_USER}\"
|
|
||||||
(${SERVICE_USER}) $ go get -v -u github.com/searxng/filtron
|
|
||||||
|
|
||||||
.. END install filtron
|
|
||||||
|
|
@ -1,53 +0,0 @@
|
||||||
.. START create user
|
|
||||||
|
|
||||||
.. tabs::
|
|
||||||
|
|
||||||
.. group-tab:: bash
|
|
||||||
|
|
||||||
.. code-block:: sh
|
|
||||||
|
|
||||||
$ sudo -H useradd --shell /bin/bash --system \\
|
|
||||||
--home-dir \"$SERVICE_HOME\" \\
|
|
||||||
--comment \"Privacy-respecting metasearch engine\" $SERVICE_USER
|
|
||||||
|
|
||||||
$ sudo -H mkdir \"$SERVICE_HOME\"
|
|
||||||
$ sudo -H chown -R \"$SERVICE_GROUP:$SERVICE_GROUP\" \"$SERVICE_HOME\"
|
|
||||||
|
|
||||||
.. END create user
|
|
||||||
|
|
||||||
.. START install go
|
|
||||||
|
|
||||||
.. tabs::
|
|
||||||
|
|
||||||
.. group-tab:: os: linux / arch: amd64
|
|
||||||
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
$ cat > \"$GO_ENV\" <<EOF
|
|
||||||
export GOPATH=${SERVICE_HOME}/go-apps
|
|
||||||
export PATH=\$PATH:${SERVICE_HOME}/local/go/bin:\$GOPATH/bin
|
|
||||||
EOF
|
|
||||||
$ sudo -i -u \"${SERVICE_USER}\"
|
|
||||||
(${SERVICE_USER}) $ echo 'source $GO_ENV' >> ~/.profile
|
|
||||||
(${SERVICE_USER}) $ mkdir ${SERVICE_HOME}/local
|
|
||||||
(${SERVICE_USER}) $ wget --progress=bar -O \"${GO_VERSION}.linux-amd64.tar.gz\" \\
|
|
||||||
\"${GO_DL_URL}/${GO_VERSION}.linux-amd64.tar.gz\"
|
|
||||||
(${SERVICE_USER}) $ tar -C ${SERVICE_HOME}/local -xzf \"${GO_VERSION}.linux-amd64.tar.gz\"
|
|
||||||
(${SERVICE_USER}) $ which go
|
|
||||||
${SERVICE_HOME}/local/go/bin/go
|
|
||||||
|
|
||||||
.. END install go
|
|
||||||
|
|
||||||
.. START install morty
|
|
||||||
|
|
||||||
.. tabs::
|
|
||||||
|
|
||||||
.. group-tab:: bash
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
$ sudo -i -u \"${SERVICE_USER}\"
|
|
||||||
(${SERVICE_USER}) $ go get -v -u github.com/asciimoo/morty
|
|
||||||
|
|
||||||
.. END install morty
|
|
||||||
|
|
@ -148,7 +148,7 @@ live build
|
||||||
Live build is like WYSIWYG. If you want to edit the documentation, its
|
Live build is like WYSIWYG. If you want to edit the documentation, its
|
||||||
recommended to use. The Makefile target ``docs.live`` builds the docs, opens
|
recommended to use. The Makefile target ``docs.live`` builds the docs, opens
|
||||||
URL in your favorite browser and rebuilds every time a reST file has been
|
URL in your favorite browser and rebuilds every time a reST file has been
|
||||||
changed.
|
changed (:ref:`make docs.clean`).
|
||||||
|
|
||||||
.. code:: sh
|
.. code:: sh
|
||||||
|
|
||||||
|
|
@ -183,3 +183,8 @@ commit and push:
|
||||||
.. code:: sh
|
.. code:: sh
|
||||||
|
|
||||||
$ make docs.clean docs.gh-pages
|
$ make docs.clean docs.gh-pages
|
||||||
|
|
||||||
|
.. attention::
|
||||||
|
|
||||||
|
If you are working in your own brand, don't forgett to adjust your
|
||||||
|
:ref:`settings brand`.
|
||||||
|
|
|
||||||
|
|
@ -1,32 +1,49 @@
|
||||||
.. _makefile:
|
.. _makefile:
|
||||||
|
|
||||||
========
|
=======================
|
||||||
Makefile
|
Makefile & ``./manage``
|
||||||
========
|
=======================
|
||||||
|
|
||||||
.. _gnu-make: https://www.gnu.org/software/make/manual/make.html#Introduction
|
.. _gnu-make: https://www.gnu.org/software/make/manual/make.html#Introduction
|
||||||
|
|
||||||
|
All relevant build and development tasks are implemented in the
|
||||||
|
:origin:`./manage <manage>` script and for CI or IDE integration a small
|
||||||
|
:origin:`Makefile` wrapper is available. If you are not familiar with
|
||||||
|
Makefiles, we recommend to read gnu-make_ introduction.
|
||||||
|
|
||||||
.. sidebar:: build environment
|
.. sidebar:: build environment
|
||||||
|
|
||||||
Before looking deeper at the targets, first read about :ref:`make
|
Before looking deeper at the targets, first read about :ref:`make
|
||||||
install`.
|
install`.
|
||||||
|
|
||||||
To install system requirements follow :ref:`buildhosts`.
|
To install developer requirements follow :ref:`buildhosts`.
|
||||||
|
|
||||||
All relevant build tasks are implemented in :origin:`manage` and for CI or
|
|
||||||
IDE integration a small ``Makefile`` wrapper is available. If you are not
|
|
||||||
familiar with Makefiles, we recommend to read gnu-make_ introduction.
|
|
||||||
|
|
||||||
The usage is simple, just type ``make {target-name}`` to *build* a target.
|
|
||||||
Calling the ``help`` target gives a first overview (``make help``):
|
|
||||||
|
|
||||||
.. program-output:: bash -c "cd ..; make --no-print-directory help"
|
|
||||||
|
|
||||||
.. contents::
|
.. contents::
|
||||||
:depth: 2
|
:depth: 2
|
||||||
:local:
|
:local:
|
||||||
:backlinks: entry
|
:backlinks: entry
|
||||||
|
|
||||||
|
The usage is simple, just type ``make {target-name}`` to *build* a target.
|
||||||
|
Calling the ``help`` target gives a first overview (``make help``):
|
||||||
|
|
||||||
|
.. tabs::
|
||||||
|
|
||||||
|
.. group-tab:: ``make``
|
||||||
|
|
||||||
|
.. program-output:: bash -c "cd ..; make --no-print-directory help"
|
||||||
|
|
||||||
|
|
||||||
|
.. group-tab:: ``./manage``
|
||||||
|
|
||||||
|
The Makefile targets are implemented for comfort, if you can do without
|
||||||
|
tab-completion and need to have a more granular control, use
|
||||||
|
:origin:`manage` without the Makefile wrappers.
|
||||||
|
|
||||||
|
.. code:: sh
|
||||||
|
|
||||||
|
$ ./manage help
|
||||||
|
|
||||||
.. _make install:
|
.. _make install:
|
||||||
|
|
||||||
Python environment (``make install``)
|
Python environment (``make install``)
|
||||||
|
|
@ -158,13 +175,36 @@ Node.js environment (``make node.env``)
|
||||||
Manager) to install latest LTS of Node.js_ locally: there is no need to
|
Manager) to install latest LTS of Node.js_ locally: there is no need to
|
||||||
install nvm_ or npm_ on your system.
|
install nvm_ or npm_ on your system.
|
||||||
|
|
||||||
Use ``make nvm.status`` to get the current status of you Node.js_ and nvm_ setup.
|
To install NVM_ and Node.js_ in once you can use :ref:`make nvm.nodejs`.
|
||||||
|
|
||||||
Here is the output you will typically get on a Ubuntu 20.04 system which serves
|
.. _make nvm:
|
||||||
only a `no longer active <https://nodejs.org/en/about/releases/>`_ Release
|
|
||||||
`Node.js v10.19.0 <https://packages.ubuntu.com/focal/nodejs>`_.
|
|
||||||
|
|
||||||
::
|
NVM ``make nvm.install nvm.status``
|
||||||
|
-----------------------------------
|
||||||
|
|
||||||
|
Use ``make nvm.status`` to get the current status of your Node.js_ and nvm_
|
||||||
|
setup.
|
||||||
|
|
||||||
|
.. tabs::
|
||||||
|
|
||||||
|
.. group-tab:: nvm.install
|
||||||
|
|
||||||
|
.. code:: sh
|
||||||
|
|
||||||
|
$ LANG=C make nvm.install
|
||||||
|
INFO: install (update) NVM at ./searxng/.nvm
|
||||||
|
INFO: clone: https://github.com/nvm-sh/nvm.git
|
||||||
|
|| Cloning into './searxng/.nvm'...
|
||||||
|
INFO: checkout v0.39.4
|
||||||
|
|| HEAD is now at 8fbf8ab v0.39.4
|
||||||
|
|
||||||
|
.. group-tab:: nvm.status (ubu2004)
|
||||||
|
|
||||||
|
Here is the output you will typically get on a Ubuntu 20.04 system which
|
||||||
|
serves only a `no longer active <https://nodejs.org/en/about/releases/>`_
|
||||||
|
Release `Node.js v10.19.0 <https://packages.ubuntu.com/focal/nodejs>`_.
|
||||||
|
|
||||||
|
.. code:: sh
|
||||||
|
|
||||||
$ make nvm.status
|
$ make nvm.status
|
||||||
INFO: Node.js is installed at /usr/bin/node
|
INFO: Node.js is installed at /usr/bin/node
|
||||||
|
|
@ -173,14 +213,11 @@ only a `no longer active <https://nodejs.org/en/about/releases/>`_ Release
|
||||||
INFO: npm is installed at /usr/bin/npm
|
INFO: npm is installed at /usr/bin/npm
|
||||||
INFO: npm is version 6.14.4
|
INFO: npm is version 6.14.4
|
||||||
WARN: NVM is not installed
|
WARN: NVM is not installed
|
||||||
INFO: to install NVM and Node.js (LTS) use: manage nvm install --lts
|
|
||||||
|
|
||||||
To install you can also use :ref:`make nvm.nodejs`
|
|
||||||
|
|
||||||
.. _make nvm.nodejs:
|
.. _make nvm.nodejs:
|
||||||
|
|
||||||
``make nvm.nodejs``
|
``make nvm.nodejs``
|
||||||
===================
|
-------------------
|
||||||
|
|
||||||
Install latest Node.js_ LTS locally (uses nvm_)::
|
Install latest Node.js_ LTS locally (uses nvm_)::
|
||||||
|
|
||||||
|
|
@ -213,10 +250,29 @@ sources of the theme need to be rebuild. You can do that by running::
|
||||||
$ make themes.all
|
$ make themes.all
|
||||||
|
|
||||||
Alternatively to ``themes.all`` you can run *live builds* of the theme you are
|
Alternatively to ``themes.all`` you can run *live builds* of the theme you are
|
||||||
modify::
|
modify (:ref:`make themes`)::
|
||||||
|
|
||||||
$ LIVE_THEME=simple make run
|
$ LIVE_THEME=simple make run
|
||||||
|
|
||||||
|
.. _make format.python:
|
||||||
|
|
||||||
|
``make format.python``
|
||||||
|
======================
|
||||||
|
|
||||||
|
Format Python sourcee code using `Black code style`_. See ``$BLACK_OPTIONS``
|
||||||
|
and ``$BLACK_TARGETS`` in :origin:`Makefile`.
|
||||||
|
|
||||||
|
.. attention::
|
||||||
|
|
||||||
|
We stuck at Black 22.12.0, please read comment in PR `Bump black from 22.12.0
|
||||||
|
to 23.1.0`_
|
||||||
|
|
||||||
|
.. _Bump black from 22.12.0 to 23.1.0:
|
||||||
|
https://github.com/searxng/searxng/pull/2159#pullrequestreview-1284094735
|
||||||
|
|
||||||
|
.. _Black code style:
|
||||||
|
https://black.readthedocs.io/en/stable/the_black_code_style/current_style.html
|
||||||
|
|
||||||
.. _make clean:
|
.. _make clean:
|
||||||
|
|
||||||
``make clean``
|
``make clean``
|
||||||
|
|
@ -237,18 +293,34 @@ calling ``make clean`` stop all processes using the :ref:`make install` or
|
||||||
|
|
||||||
.. _make docs:
|
.. _make docs:
|
||||||
|
|
||||||
``make docs docs.autobuild docs.clean``
|
``make docs``
|
||||||
=======================================
|
=============
|
||||||
|
|
||||||
|
Target ``docs`` builds the documentation:
|
||||||
|
|
||||||
|
.. code:: bash
|
||||||
|
|
||||||
|
$ make docs
|
||||||
|
HTML ./docs --> file://
|
||||||
|
DOCS build build/docs/includes
|
||||||
|
...
|
||||||
|
The HTML pages are in dist/docs.
|
||||||
|
|
||||||
|
.. _make docs.clean:
|
||||||
|
|
||||||
|
``make docs.clean docs.live``
|
||||||
|
----------------------------------
|
||||||
|
|
||||||
We describe the usage of the ``doc.*`` targets in the :ref:`How to contribute /
|
We describe the usage of the ``doc.*`` targets in the :ref:`How to contribute /
|
||||||
Documentation <contrib docs>` section. If you want to edit the documentation
|
Documentation <contrib docs>` section. If you want to edit the documentation
|
||||||
read our :ref:`make docs.live` section. If you are working in your own brand,
|
read our :ref:`make docs.live` section. If you are working in your own brand,
|
||||||
adjust your :ref:`settings brand`.
|
adjust your :ref:`settings brand`.
|
||||||
|
|
||||||
|
|
||||||
.. _make docs.gh-pages:
|
.. _make docs.gh-pages:
|
||||||
|
|
||||||
``make docs.gh-pages``
|
``make docs.gh-pages``
|
||||||
======================
|
----------------------
|
||||||
|
|
||||||
To deploy on github.io first adjust your :ref:`settings brand`. For any
|
To deploy on github.io first adjust your :ref:`settings brand`. For any
|
||||||
further read :ref:`deploy on github.io`.
|
further read :ref:`deploy on github.io`.
|
||||||
|
|
@ -261,17 +333,17 @@ further read :ref:`deploy on github.io`.
|
||||||
Runs a series of tests: :ref:`make test.pylint`, ``test.pep8``, ``test.unit``
|
Runs a series of tests: :ref:`make test.pylint`, ``test.pep8``, ``test.unit``
|
||||||
and ``test.robot``. You can run tests selective, e.g.::
|
and ``test.robot``. You can run tests selective, e.g.::
|
||||||
|
|
||||||
$ make test.pep8 test.unit test.sh
|
$ make test.pep8 test.unit test.shell
|
||||||
TEST test.pep8 OK
|
TEST test.pep8 OK
|
||||||
...
|
...
|
||||||
TEST test.unit OK
|
TEST test.unit OK
|
||||||
...
|
...
|
||||||
TEST test.sh OK
|
TEST test.shell OK
|
||||||
|
|
||||||
.. _make test.shell:
|
.. _make test.shell:
|
||||||
|
|
||||||
``make test.shell``
|
``make test.shell``
|
||||||
===================
|
-------------------
|
||||||
|
|
||||||
:ref:`sh lint` / if you have changed some bash scripting run this test before
|
:ref:`sh lint` / if you have changed some bash scripting run this test before
|
||||||
commit.
|
commit.
|
||||||
|
|
@ -279,7 +351,7 @@ commit.
|
||||||
.. _make test.pylint:
|
.. _make test.pylint:
|
||||||
|
|
||||||
``make test.pylint``
|
``make test.pylint``
|
||||||
====================
|
--------------------
|
||||||
|
|
||||||
.. _Pylint: https://www.pylint.org/
|
.. _Pylint: https://www.pylint.org/
|
||||||
|
|
||||||
|
|
@ -289,8 +361,8 @@ found in project's root folder :origin:`.pylintrc`.
|
||||||
|
|
||||||
.. _make search.checker:
|
.. _make search.checker:
|
||||||
|
|
||||||
``search.checker.{engine name}``
|
``make search.checker.{engine name}``
|
||||||
================================
|
=====================================
|
||||||
|
|
||||||
To check all engines::
|
To check all engines::
|
||||||
|
|
||||||
|
|
@ -318,3 +390,63 @@ To filter out HTTP redirects (3xx_)::
|
||||||
https://news.google.com:443 "GET /search?q=computer&hl=en&lr=lang_en&ie=utf8&oe=utf8&ceid=US%3Aen&gl=US HTTP/1.1" 302 0
|
https://news.google.com:443 "GET /search?q=computer&hl=en&lr=lang_en&ie=utf8&oe=utf8&ceid=US%3Aen&gl=US HTTP/1.1" 302 0
|
||||||
https://news.google.com:443 "GET /search?q=computer&hl=en-US&lr=lang_en&ie=utf8&oe=utf8&ceid=US:en&gl=US HTTP/1.1" 200 None
|
https://news.google.com:443 "GET /search?q=computer&hl=en-US&lr=lang_en&ie=utf8&oe=utf8&ceid=US:en&gl=US HTTP/1.1" 200 None
|
||||||
--
|
--
|
||||||
|
|
||||||
|
.. _make themes:
|
||||||
|
|
||||||
|
``make themes.*``
|
||||||
|
=================
|
||||||
|
|
||||||
|
.. sidebar:: further read
|
||||||
|
|
||||||
|
- :ref:`devquickstart`
|
||||||
|
|
||||||
|
The :origin:`Makefile` targets ``make theme.*`` cover common tasks to build the
|
||||||
|
theme(s). The ``./manage themes.*`` command line can be used to convenient run
|
||||||
|
common theme build tasks.
|
||||||
|
|
||||||
|
.. program-output:: bash -c "cd ..; ./manage themes.help"
|
||||||
|
|
||||||
|
To get live builds while modifying CSS & JS use (:ref:`make run`):
|
||||||
|
|
||||||
|
.. code:: sh
|
||||||
|
|
||||||
|
$ LIVE_THEME=simple make run
|
||||||
|
|
||||||
|
.. _make static.build:
|
||||||
|
|
||||||
|
``make static.build.*``
|
||||||
|
=======================
|
||||||
|
|
||||||
|
.. sidebar:: further read
|
||||||
|
|
||||||
|
- :ref:`devquickstart`
|
||||||
|
|
||||||
|
The :origin:`Makefile` targets ``static.build.*`` cover common tasks to build (a
|
||||||
|
commit of) the static files. The ``./manage static.build..*`` command line
|
||||||
|
can be used to convenient run common build tasks of the satic files.
|
||||||
|
|
||||||
|
.. program-output:: bash -c "cd ..; ./manage static.help"
|
||||||
|
|
||||||
|
|
||||||
|
.. _manage redis.help:
|
||||||
|
|
||||||
|
``./manage redis.help``
|
||||||
|
=======================
|
||||||
|
|
||||||
|
The ``./manage redis.*`` command line can be used to convenient run common Redis
|
||||||
|
tasks (:ref:`Redis developer notes`).
|
||||||
|
|
||||||
|
.. program-output:: bash -c "cd ..; ./manage redis.help"
|
||||||
|
|
||||||
|
|
||||||
|
.. _manage go.help:
|
||||||
|
|
||||||
|
``./manage go.help``
|
||||||
|
====================
|
||||||
|
|
||||||
|
The ``./manage go.*`` command line can be used to convenient run common `go
|
||||||
|
(wiki)`_ tasks.
|
||||||
|
|
||||||
|
.. _go (wiki): https://en.wikipedia.org/wiki/Go_(programming_language)
|
||||||
|
|
||||||
|
.. program-output:: bash -c "cd ..; ./manage go.help"
|
||||||
|
|
|
||||||
|
|
@ -7,8 +7,16 @@ Development Quickstart
|
||||||
.. _npm: https://www.npmjs.com/
|
.. _npm: https://www.npmjs.com/
|
||||||
.. _Node.js: https://nodejs.org/
|
.. _Node.js: https://nodejs.org/
|
||||||
|
|
||||||
SearXNG loves developers, just clone and start hacking. All the rest is done for
|
|
||||||
you simply by using :ref:`make <makefile>`.
|
.. sidebar:: further read
|
||||||
|
|
||||||
|
- :ref:`makefile`
|
||||||
|
- :ref:`buildhosts`
|
||||||
|
|
||||||
|
SearXNG loves developers; Developers do not need to worry about tool chains, the
|
||||||
|
usual developer tasks can be comfortably executed via :ref:`make <makefile>`.
|
||||||
|
|
||||||
|
Don't hesitate, just clone SearXNG's sources and start hacking right now ..
|
||||||
|
|
||||||
.. code:: bash
|
.. code:: bash
|
||||||
|
|
||||||
|
|
@ -18,25 +26,23 @@ Here is how a minimal workflow looks like:
|
||||||
|
|
||||||
1. *start* hacking
|
1. *start* hacking
|
||||||
2. *run* your code: :ref:`make run`
|
2. *run* your code: :ref:`make run`
|
||||||
3. *test* your code: :ref:`make test`
|
3. *format & test* your code: :ref:`make format.python` and :ref:`make test`
|
||||||
|
|
||||||
If you think at some point something fails, go back to *start*. Otherwise,
|
If you think at some point something fails, go back to *start*. Otherwise,
|
||||||
choose a meaningful commit message and we are happy to receive your pull
|
choose a meaningful commit message and we are happy to receive your pull
|
||||||
request. To not end in *wild west* we have some directives, please pay attention
|
request. To not end in *wild west* we have some directives, please pay attention
|
||||||
to our ":ref:`how to contribute`" guideline.
|
to our ":ref:`how to contribute`" guideline.
|
||||||
|
|
||||||
If you implement themes, you will need to setup a :ref:`make node.env` once:
|
.. sidebar:: further read
|
||||||
|
|
||||||
.. code:: bash
|
- :ref:`make nvm`
|
||||||
|
- :ref:`make themes`
|
||||||
|
|
||||||
make node.env
|
If you implement themes, you will need to setup a :ref:`Node.js environment
|
||||||
|
<make node.env>`: ``make node.env``
|
||||||
|
|
||||||
Before you call *make run* (2.), you need to compile the modified styles and
|
Before you call *make run* (2.), you need to compile the modified styles and
|
||||||
JavaScript:
|
JavaScript: ``make themes.all``
|
||||||
|
|
||||||
.. code:: bash
|
|
||||||
|
|
||||||
make themes.all
|
|
||||||
|
|
||||||
Alternatively you can also compile selective the theme you have modified,
|
Alternatively you can also compile selective the theme you have modified,
|
||||||
e.g. the *simple* theme.
|
e.g. the *simple* theme.
|
||||||
|
|
@ -49,6 +55,10 @@ e.g. the *simple* theme.
|
||||||
|
|
||||||
To get live builds while modifying CSS & JS use: ``LIVE_THEME=simple make run``
|
To get live builds while modifying CSS & JS use: ``LIVE_THEME=simple make run``
|
||||||
|
|
||||||
|
.. sidebar:: further read
|
||||||
|
|
||||||
|
- :ref:`make static.build`
|
||||||
|
|
||||||
If you finished your *tests* you can start to commit your changes. To separate
|
If you finished your *tests* you can start to commit your changes. To separate
|
||||||
the modified source code from the build products first run:
|
the modified source code from the build products first run:
|
||||||
|
|
||||||
|
|
|
||||||
487
manage
487
manage
|
|
@ -11,8 +11,23 @@ source "$(dirname "${BASH_SOURCE[0]}")/utils/lib.sh"
|
||||||
# shellcheck source=utils/lib.sh
|
# shellcheck source=utils/lib.sh
|
||||||
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_nvm.sh"
|
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_nvm.sh"
|
||||||
|
|
||||||
# shellcheck source=utils/lib_static.sh
|
# shellcheck source=utils/lib_sxng_data.sh
|
||||||
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_static.sh"
|
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_data.sh"
|
||||||
|
|
||||||
|
# shellcheck source=utils/lib_sxng_weblate.sh
|
||||||
|
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_weblate.sh"
|
||||||
|
|
||||||
|
# shellcheck source=utils/lib_sxng_static.sh
|
||||||
|
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_static.sh"
|
||||||
|
|
||||||
|
# shellcheck source=utils/lib_sxng_node.sh
|
||||||
|
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_node.sh"
|
||||||
|
|
||||||
|
# shellcheck source=utils/lib_sxng_themes.sh
|
||||||
|
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_themes.sh"
|
||||||
|
|
||||||
|
# shellcheck source=utils/lib_sxng_test.sh
|
||||||
|
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_test.sh"
|
||||||
|
|
||||||
# shellcheck source=utils/lib_go.sh
|
# shellcheck source=utils/lib_go.sh
|
||||||
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_go.sh"
|
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_go.sh"
|
||||||
|
|
@ -27,9 +42,8 @@ PATH="${REPO_ROOT}/node_modules/.bin:${PATH}"
|
||||||
PYOBJECTS="searx"
|
PYOBJECTS="searx"
|
||||||
PY_SETUP_EXTRAS='[test]'
|
PY_SETUP_EXTRAS='[test]'
|
||||||
GECKODRIVER_VERSION="v0.30.0"
|
GECKODRIVER_VERSION="v0.30.0"
|
||||||
export NODE_MINIMUM_VERSION="16.13.0"
|
|
||||||
# SPHINXOPTS=
|
# SPHINXOPTS=
|
||||||
BLACK_OPTIONS=("--target-version" "py37" "--line-length" "120" "--skip-string-normalization")
|
BLACK_OPTIONS=("--target-version" "py311" "--line-length" "120" "--skip-string-normalization")
|
||||||
BLACK_TARGETS=("--exclude" "searx/static,searx/languages.py" "--include" 'searxng.msg|\.pyi?$' "searx" "searxng_extra" "tests")
|
BLACK_TARGETS=("--exclude" "searx/static,searx/languages.py" "--include" 'searxng.msg|\.pyi?$' "searx" "searxng_extra" "tests")
|
||||||
|
|
||||||
_dev_redis_sock="/usr/local/searxng-redis/run/redis.sock"
|
_dev_redis_sock="/usr/local/searxng-redis/run/redis.sock"
|
||||||
|
|
@ -73,13 +87,6 @@ buildenv:
|
||||||
rebuild ./utils/brand.env
|
rebuild ./utils/brand.env
|
||||||
webapp.:
|
webapp.:
|
||||||
run : run developer instance
|
run : run developer instance
|
||||||
weblate.:
|
|
||||||
push.translations: push translation changes from SearXNG to Weblate's counterpart
|
|
||||||
to.translations: Update 'translations' branch with last additions from Weblate.
|
|
||||||
data.:
|
|
||||||
all : update searx/sxng_locales.py and searx/data/*
|
|
||||||
traits : update searx/data/engine_traits.json & searx/sxng_locales.py
|
|
||||||
useragents: update searx/data/useragents.json with the most recent versions of Firefox
|
|
||||||
docs.:
|
docs.:
|
||||||
html : build HTML documentation
|
html : build HTML documentation
|
||||||
live : autobuild HTML documentation while editing
|
live : autobuild HTML documentation while editing
|
||||||
|
|
@ -96,10 +103,6 @@ redis:
|
||||||
build : build redis binaries at $(redis._get_dist)
|
build : build redis binaries at $(redis._get_dist)
|
||||||
install : create user (${REDIS_USER}) and install systemd service (${REDIS_SERVICE_NAME})
|
install : create user (${REDIS_USER}) and install systemd service (${REDIS_SERVICE_NAME})
|
||||||
help : show more redis commands
|
help : show more redis commands
|
||||||
node.:
|
|
||||||
env : download & install SearXNG's npm dependencies locally
|
|
||||||
env.dev : download & install developer and CI tools
|
|
||||||
clean : drop locally npm installations
|
|
||||||
py.:
|
py.:
|
||||||
build : Build python packages at ./${PYDIST}
|
build : Build python packages at ./${PYDIST}
|
||||||
clean : delete virtualenv and intermediate py files
|
clean : delete virtualenv and intermediate py files
|
||||||
|
|
@ -112,24 +115,16 @@ pypi.upload:
|
||||||
Upload python packages to PyPi (to test use pypi.upload.test)
|
Upload python packages to PyPi (to test use pypi.upload.test)
|
||||||
format.:
|
format.:
|
||||||
python : format Python code source using black
|
python : format Python code source using black
|
||||||
test.:
|
|
||||||
yamllint : lint YAML files (YAMLLINT_FILES)
|
|
||||||
pylint : lint PYLINT_FILES, searx/engines, searx & tests
|
|
||||||
pyright : static type check of python sources
|
|
||||||
black : check black code format
|
|
||||||
unit : run unit tests
|
|
||||||
coverage : run unit tests with coverage
|
|
||||||
robot : run robot test
|
|
||||||
rst : test .rst files incl. README.rst
|
|
||||||
clean : clean intermediate test stuff
|
|
||||||
themes.:
|
|
||||||
all : build all themes
|
|
||||||
simple : build simple theme
|
|
||||||
pygments.:
|
pygments.:
|
||||||
less : build LESS files for pygments
|
less : build LESS files for pygments
|
||||||
EOF
|
EOF
|
||||||
go.help
|
go.help
|
||||||
static_help
|
node.help
|
||||||
|
weblate.help
|
||||||
|
data.help
|
||||||
|
test.help
|
||||||
|
themes.help
|
||||||
|
static.help
|
||||||
cat <<EOF
|
cat <<EOF
|
||||||
environment ...
|
environment ...
|
||||||
SEARXNG_REDIS_URL : ${SEARXNG_REDIS_URL}
|
SEARXNG_REDIS_URL : ${SEARXNG_REDIS_URL}
|
||||||
|
|
@ -183,258 +178,6 @@ buildenv() {
|
||||||
return "${PIPESTATUS[0]}"
|
return "${PIPESTATUS[0]}"
|
||||||
}
|
}
|
||||||
|
|
||||||
TRANSLATIONS_WORKTREE="$CACHE/translations"
|
|
||||||
|
|
||||||
weblate.translations.worktree() {
|
|
||||||
|
|
||||||
# Create git worktree ${TRANSLATIONS_WORKTREE} and checkout branch
|
|
||||||
# 'translations' from Weblate's counterpart (weblate) of the SearXNG
|
|
||||||
# (origin).
|
|
||||||
#
|
|
||||||
# remote weblate https://translate.codeberg.org/git/searxng/searxng/
|
|
||||||
|
|
||||||
( set -e
|
|
||||||
if ! git remote get-url weblate 2> /dev/null; then
|
|
||||||
git remote add weblate https://translate.codeberg.org/git/searxng/searxng/
|
|
||||||
fi
|
|
||||||
if [ -d "${TRANSLATIONS_WORKTREE}" ]; then
|
|
||||||
pushd .
|
|
||||||
cd "${TRANSLATIONS_WORKTREE}"
|
|
||||||
git reset --hard HEAD
|
|
||||||
git pull origin translations
|
|
||||||
popd
|
|
||||||
else
|
|
||||||
mkdir -p "${TRANSLATIONS_WORKTREE}"
|
|
||||||
git worktree add "${TRANSLATIONS_WORKTREE}" translations
|
|
||||||
fi
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
weblate.to.translations() {
|
|
||||||
|
|
||||||
# Update 'translations' branch of SearXNG (origin) with last additions from
|
|
||||||
# Weblate.
|
|
||||||
|
|
||||||
# 1. Check if Weblate is locked, if not die with error message
|
|
||||||
# 2. On Weblate's counterpart (weblate), pull master and translations branch
|
|
||||||
# from SearXNG (origin).
|
|
||||||
# 3. Commit changes made in a Weblate object on Weblate's counterpart
|
|
||||||
# (weblate).
|
|
||||||
# 4. In translations worktree, merge changes of branch 'translations' from
|
|
||||||
# remote 'weblate' and push it on branch 'translations' of 'origin'
|
|
||||||
|
|
||||||
( set -e
|
|
||||||
pyenv.activate
|
|
||||||
if [ "$(wlc lock-status)" != "locked: True" ]; then
|
|
||||||
die 1 "weblate must be locked, currently: $(wlc lock-status)"
|
|
||||||
fi
|
|
||||||
# weblate: commit pending changes
|
|
||||||
wlc pull
|
|
||||||
wlc commit
|
|
||||||
|
|
||||||
# get the translations in a worktree
|
|
||||||
weblate.translations.worktree
|
|
||||||
|
|
||||||
pushd "${TRANSLATIONS_WORKTREE}"
|
|
||||||
git remote update weblate
|
|
||||||
git merge weblate/translations
|
|
||||||
git push
|
|
||||||
popd
|
|
||||||
)
|
|
||||||
dump_return $?
|
|
||||||
}
|
|
||||||
|
|
||||||
weblate.translations.commit() {
|
|
||||||
|
|
||||||
# Update 'translations' branch of SearXNG (origin) with last additions from
|
|
||||||
# Weblate. Copy the changes to the master branch, compile translations and
|
|
||||||
# create a commit in the local branch (master)
|
|
||||||
|
|
||||||
local existing_commit_hash commit_body commit_message exitcode
|
|
||||||
( set -e
|
|
||||||
pyenv.activate
|
|
||||||
# lock change on weblate
|
|
||||||
wlc lock
|
|
||||||
|
|
||||||
# get translations branch in git worktree (TRANSLATIONS_WORKTREE)
|
|
||||||
weblate.translations.worktree
|
|
||||||
existing_commit_hash=$(cd "${TRANSLATIONS_WORKTREE}"; git log -n1 --pretty=format:'%h')
|
|
||||||
|
|
||||||
# pull weblate commits
|
|
||||||
weblate.to.translations
|
|
||||||
|
|
||||||
# copy the changes to the master branch
|
|
||||||
cp -rv --preserve=mode,timestamps "${TRANSLATIONS_WORKTREE}/searx/translations" "searx"
|
|
||||||
|
|
||||||
# compile translations
|
|
||||||
build_msg BABEL 'compile translation catalogs into binary MO files'
|
|
||||||
pybabel compile --statistics \
|
|
||||||
-d "searx/translations"
|
|
||||||
# git add/commit (no push)
|
|
||||||
commit_body=$(cd "${TRANSLATIONS_WORKTREE}"; git log --pretty=format:'%h - %as - %aN <%ae>' "${existing_commit_hash}..HEAD")
|
|
||||||
commit_message=$(echo -e "[translations] update from Weblate\n\n${commit_body}")
|
|
||||||
git add searx/translations
|
|
||||||
git commit -m "${commit_message}"
|
|
||||||
)
|
|
||||||
exitcode=$?
|
|
||||||
( # make sure to always unlock weblate
|
|
||||||
set -e
|
|
||||||
pyenv.cmd wlc unlock
|
|
||||||
)
|
|
||||||
dump_return $exitcode
|
|
||||||
}
|
|
||||||
|
|
||||||
weblate.push.translations() {
|
|
||||||
|
|
||||||
# Push *translation changes* from SearXNG (origin) to Weblate's counterpart
|
|
||||||
# (weblate).
|
|
||||||
|
|
||||||
# In branch master of SearXNG (origin) check for meaningful changes in
|
|
||||||
# folder 'searx/translations', commit changes on branch 'translations' and
|
|
||||||
# at least, pull updated branches on Weblate's counterpart (weblate).
|
|
||||||
|
|
||||||
# 1. Create git worktree ${TRANSLATIONS_WORKTREE} and checkout branch
|
|
||||||
# 'translations' from remote 'weblate'.
|
|
||||||
# 2. Stop if there is no meaningful change in the 'master' branch (origin),
|
|
||||||
# compared to the 'translations' branch (weblate), otherwise ...
|
|
||||||
# 3. Update 'translations' branch of SearXNG (origin) with last additions
|
|
||||||
# from Weblate.
|
|
||||||
# 5. Notify Weblate to pull updated 'master' & 'translations' branch.
|
|
||||||
|
|
||||||
local messages_pot diff_messages_pot last_commit_hash last_commit_detail \
|
|
||||||
exitcode
|
|
||||||
messages_pot="${TRANSLATIONS_WORKTREE}/searx/translations/messages.pot"
|
|
||||||
( set -e
|
|
||||||
pyenv.activate
|
|
||||||
# get translations branch in git worktree (TRANSLATIONS_WORKTREE)
|
|
||||||
weblate.translations.worktree
|
|
||||||
|
|
||||||
# update messages.pot in the master branch
|
|
||||||
build_msg BABEL 'extract messages from source files and generate POT file'
|
|
||||||
pybabel extract -F babel.cfg \
|
|
||||||
-o "${messages_pot}" \
|
|
||||||
"searx/"
|
|
||||||
|
|
||||||
# stop if there is no meaningful change in the master branch
|
|
||||||
diff_messages_pot=$(cd "${TRANSLATIONS_WORKTREE}";\
|
|
||||||
git diff -- "searx/translations/messages.pot")
|
|
||||||
if ! echo "$diff_messages_pot" | grep -qE "[\+\-](msgid|msgstr)"; then
|
|
||||||
build_msg BABEL 'no changes detected, exiting'
|
|
||||||
return 42
|
|
||||||
fi
|
|
||||||
return 0
|
|
||||||
)
|
|
||||||
exitcode=$?
|
|
||||||
if [ "$exitcode" -eq 42 ]; then
|
|
||||||
return 0
|
|
||||||
fi
|
|
||||||
if [ "$exitcode" -gt 0 ]; then
|
|
||||||
return $exitcode
|
|
||||||
fi
|
|
||||||
(
|
|
||||||
set -e
|
|
||||||
pyenv.activate
|
|
||||||
|
|
||||||
# lock change on weblate
|
|
||||||
# weblate may add commit(s) since the call to "weblate.translations.worktree".
|
|
||||||
# this is not a problem because after this line, "weblate.to.translations"
|
|
||||||
# calls again "weblate.translations.worktree" which calls "git pull"
|
|
||||||
wlc lock
|
|
||||||
|
|
||||||
# save messages.pot in the translations branch for later
|
|
||||||
pushd "${TRANSLATIONS_WORKTREE}"
|
|
||||||
git stash push
|
|
||||||
popd
|
|
||||||
|
|
||||||
# merge weblate commits into the translations branch
|
|
||||||
weblate.to.translations
|
|
||||||
|
|
||||||
# restore messages.pot in the translations branch
|
|
||||||
pushd "${TRANSLATIONS_WORKTREE}"
|
|
||||||
git stash pop
|
|
||||||
popd
|
|
||||||
|
|
||||||
# update messages.po files in the master branch
|
|
||||||
build_msg BABEL 'update existing message catalogs from POT file'
|
|
||||||
pybabel update -N \
|
|
||||||
-i "${messages_pot}" \
|
|
||||||
-d "${TRANSLATIONS_WORKTREE}/searx/translations"
|
|
||||||
|
|
||||||
# git add/commit/push
|
|
||||||
last_commit_hash=$(git log -n1 --pretty=format:'%h')
|
|
||||||
last_commit_detail=$(git log -n1 --pretty=format:'%h - %as - %aN <%ae>' "${last_commit_hash}")
|
|
||||||
|
|
||||||
pushd "${TRANSLATIONS_WORKTREE}"
|
|
||||||
git add searx/translations
|
|
||||||
git commit \
|
|
||||||
-m "[translations] update messages.pot and messages.po files" \
|
|
||||||
-m "From ${last_commit_detail}"
|
|
||||||
git push
|
|
||||||
popd
|
|
||||||
|
|
||||||
# notify weblate to pull updated master & translations branch
|
|
||||||
wlc pull
|
|
||||||
)
|
|
||||||
exitcode=$?
|
|
||||||
( # make sure to always unlock weblate
|
|
||||||
set -e
|
|
||||||
pyenv.activate
|
|
||||||
wlc unlock
|
|
||||||
)
|
|
||||||
dump_return $exitcode
|
|
||||||
}
|
|
||||||
|
|
||||||
data.all() {
|
|
||||||
( set -e
|
|
||||||
|
|
||||||
pyenv.activate
|
|
||||||
data.traits
|
|
||||||
data.useragents
|
|
||||||
|
|
||||||
build_msg DATA "update searx/data/osm_keys_tags.json"
|
|
||||||
pyenv.cmd python searxng_extra/update/update_osm_keys_tags.py
|
|
||||||
build_msg DATA "update searx/data/ahmia_blacklist.txt"
|
|
||||||
python searxng_extra/update/update_ahmia_blacklist.py
|
|
||||||
build_msg DATA "update searx/data/wikidata_units.json"
|
|
||||||
python searxng_extra/update/update_wikidata_units.py
|
|
||||||
build_msg DATA "update searx/data/currencies.json"
|
|
||||||
python searxng_extra/update/update_currencies.py
|
|
||||||
build_msg DATA "update searx/data/external_bangs.json"
|
|
||||||
python searxng_extra/update/update_external_bangs.py
|
|
||||||
build_msg DATA "update searx/data/engine_descriptions.json"
|
|
||||||
python searxng_extra/update/update_engine_descriptions.py
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
data.traits() {
|
|
||||||
( set -e
|
|
||||||
pyenv.activate
|
|
||||||
build_msg DATA "update searx/data/engine_traits.json"
|
|
||||||
python searxng_extra/update/update_engine_traits.py
|
|
||||||
build_msg ENGINES "update searx/sxng_locales.py"
|
|
||||||
)
|
|
||||||
dump_return $?
|
|
||||||
}
|
|
||||||
|
|
||||||
data.useragents() {
|
|
||||||
build_msg DATA "update searx/data/useragents.json"
|
|
||||||
pyenv.cmd python searxng_extra/update/update_firefox_version.py
|
|
||||||
dump_return $?
|
|
||||||
}
|
|
||||||
|
|
||||||
docs.prebuild() {
|
|
||||||
build_msg DOCS "build ${DOCS_BUILD}/includes"
|
|
||||||
(
|
|
||||||
set -e
|
|
||||||
[ "$VERBOSE" = "1" ] && set -x
|
|
||||||
mkdir -p "${DOCS_BUILD}/includes"
|
|
||||||
./utils/searxng.sh searxng.doc.rst > "${DOCS_BUILD}/includes/searxng.rst"
|
|
||||||
pyenv.cmd searxng_extra/docs_prebuild
|
|
||||||
)
|
|
||||||
dump_return $?
|
|
||||||
}
|
|
||||||
|
|
||||||
docker.push() {
|
docker.push() {
|
||||||
docker.build push
|
docker.build push
|
||||||
}
|
}
|
||||||
|
|
@ -554,44 +297,6 @@ gecko.driver() {
|
||||||
dump_return $?
|
dump_return $?
|
||||||
}
|
}
|
||||||
|
|
||||||
nodejs.ensure() {
|
|
||||||
if ! nvm.min_node "${NODE_MINIMUM_VERSION}"; then
|
|
||||||
info_msg "install Node.js by NVM"
|
|
||||||
nvm.nodejs
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
node.env() {
|
|
||||||
nodejs.ensure
|
|
||||||
( set -e
|
|
||||||
build_msg INSTALL "./searx/static/themes/simple/package.json"
|
|
||||||
npm --prefix searx/static/themes/simple install
|
|
||||||
)
|
|
||||||
dump_return $?
|
|
||||||
}
|
|
||||||
|
|
||||||
node.env.dev() {
|
|
||||||
nodejs.ensure
|
|
||||||
build_msg INSTALL "./package.json: developer and CI tools"
|
|
||||||
npm install
|
|
||||||
}
|
|
||||||
|
|
||||||
node.clean() {
|
|
||||||
if ! required_commands npm 2>/dev/null; then
|
|
||||||
build_msg CLEAN "npm is not installed / ignore npm dependencies"
|
|
||||||
return 0
|
|
||||||
fi
|
|
||||||
build_msg CLEAN "themes -- locally installed npm dependencies"
|
|
||||||
( set -e
|
|
||||||
npm --prefix searx/static/themes/simple run clean
|
|
||||||
)
|
|
||||||
build_msg CLEAN "locally installed developer and CI tools"
|
|
||||||
( set -e
|
|
||||||
npm --prefix . run clean
|
|
||||||
)
|
|
||||||
dump_return $?
|
|
||||||
}
|
|
||||||
|
|
||||||
pygments.less() {
|
pygments.less() {
|
||||||
build_msg PYGMENTS "searxng_extra/update/update_pygments.py"
|
build_msg PYGMENTS "searxng_extra/update/update_pygments.py"
|
||||||
if ! pyenv.cmd python searxng_extra/update/update_pygments.py; then
|
if ! pyenv.cmd python searxng_extra/update/update_pygments.py; then
|
||||||
|
|
@ -674,150 +379,6 @@ format.python() {
|
||||||
dump_return $?
|
dump_return $?
|
||||||
}
|
}
|
||||||
|
|
||||||
test.yamllint() {
|
|
||||||
build_msg TEST "[yamllint] \$YAMLLINT_FILES"
|
|
||||||
pyenv.cmd yamllint --strict --format parsable "${YAMLLINT_FILES[@]}"
|
|
||||||
dump_return $?
|
|
||||||
}
|
|
||||||
|
|
||||||
test.pylint() {
|
|
||||||
# shellcheck disable=SC2086
|
|
||||||
( set -e
|
|
||||||
build_msg TEST "[pylint] \$PYLINT_FILES"
|
|
||||||
pyenv.activate
|
|
||||||
python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \
|
|
||||||
--additional-builtins="${PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES}" \
|
|
||||||
"${PYLINT_FILES[@]}"
|
|
||||||
|
|
||||||
build_msg TEST "[pylint] searx/engines"
|
|
||||||
python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \
|
|
||||||
--disable="${PYLINT_SEARXNG_DISABLE_OPTION}" \
|
|
||||||
--additional-builtins="${PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES}" \
|
|
||||||
searx/engines
|
|
||||||
|
|
||||||
build_msg TEST "[pylint] searx tests"
|
|
||||||
python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \
|
|
||||||
--disable="${PYLINT_SEARXNG_DISABLE_OPTION}" \
|
|
||||||
--ignore=searx/engines \
|
|
||||||
searx tests
|
|
||||||
)
|
|
||||||
dump_return $?
|
|
||||||
}
|
|
||||||
|
|
||||||
test.pyright() {
|
|
||||||
build_msg TEST "[pyright] static type check of python sources"
|
|
||||||
node.env.dev
|
|
||||||
# We run Pyright in the virtual environment because Pyright
|
|
||||||
# executes "python" to determine the Python version.
|
|
||||||
build_msg TEST "[pyright] suppress warnings related to intentional monkey patching"
|
|
||||||
pyenv.cmd npx --no-install pyright -p pyrightconfig-ci.json \
|
|
||||||
| grep -v ".py$" \
|
|
||||||
| grep -v '/engines/.*.py.* - warning: "logger" is not defined'\
|
|
||||||
| grep -v '/plugins/.*.py.* - error: "logger" is not defined'\
|
|
||||||
| grep -v '/engines/.*.py.* - warning: "supported_languages" is not defined' \
|
|
||||||
| grep -v '/engines/.*.py.* - warning: "language_aliases" is not defined' \
|
|
||||||
| grep -v '/engines/.*.py.* - warning: "categories" is not defined'
|
|
||||||
dump_return $?
|
|
||||||
}
|
|
||||||
|
|
||||||
test.black() {
|
|
||||||
build_msg TEST "[black] \$BLACK_TARGETS"
|
|
||||||
pyenv.cmd black --check --diff "${BLACK_OPTIONS[@]}" "${BLACK_TARGETS[@]}"
|
|
||||||
dump_return $?
|
|
||||||
}
|
|
||||||
|
|
||||||
test.unit() {
|
|
||||||
build_msg TEST 'tests/unit'
|
|
||||||
pyenv.cmd python -m nose2 -s tests/unit
|
|
||||||
dump_return $?
|
|
||||||
}
|
|
||||||
|
|
||||||
test.coverage() {
|
|
||||||
build_msg TEST 'unit test coverage'
|
|
||||||
( set -e
|
|
||||||
pyenv.activate
|
|
||||||
python -m nose2 -C --log-capture --with-coverage --coverage searx -s tests/unit
|
|
||||||
coverage report
|
|
||||||
coverage html
|
|
||||||
)
|
|
||||||
dump_return $?
|
|
||||||
}
|
|
||||||
|
|
||||||
test.robot() {
|
|
||||||
build_msg TEST 'robot'
|
|
||||||
gecko.driver
|
|
||||||
PYTHONPATH=. pyenv.cmd python -m tests.robot
|
|
||||||
dump_return $?
|
|
||||||
}
|
|
||||||
|
|
||||||
test.rst() {
|
|
||||||
build_msg TEST "[reST markup] ${RST_FILES[*]}"
|
|
||||||
for rst in "${RST_FILES[@]}"; do
|
|
||||||
pyenv.cmd rst2html.py --halt error "$rst" > /dev/null || die 42 "fix issue in $rst"
|
|
||||||
done
|
|
||||||
}
|
|
||||||
|
|
||||||
test.pybabel() {
|
|
||||||
TEST_BABEL_FOLDER="build/test/pybabel"
|
|
||||||
build_msg TEST "[extract messages] pybabel"
|
|
||||||
mkdir -p "${TEST_BABEL_FOLDER}"
|
|
||||||
pyenv.cmd pybabel extract -F babel.cfg -o "${TEST_BABEL_FOLDER}/messages.pot" searx
|
|
||||||
}
|
|
||||||
|
|
||||||
test.clean() {
|
|
||||||
build_msg CLEAN "test stuff"
|
|
||||||
rm -rf geckodriver.log .coverage coverage/
|
|
||||||
dump_return $?
|
|
||||||
}
|
|
||||||
|
|
||||||
themes.all() {
|
|
||||||
( set -e
|
|
||||||
pygments.less
|
|
||||||
node.env
|
|
||||||
themes.simple
|
|
||||||
)
|
|
||||||
dump_return $?
|
|
||||||
}
|
|
||||||
|
|
||||||
themes.live() {
|
|
||||||
local LIVE_THEME="${LIVE_THEME:-${1}}"
|
|
||||||
case "${LIVE_THEME}" in
|
|
||||||
simple)
|
|
||||||
theme="searx/static/themes/${LIVE_THEME}"
|
|
||||||
;;
|
|
||||||
'')
|
|
||||||
die_caller 42 "missing theme argument"
|
|
||||||
;;
|
|
||||||
*)
|
|
||||||
die_caller 42 "unknown theme '${LIVE_THEME}' // [simple]'"
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
build_msg GRUNT "theme: $1 (live build)"
|
|
||||||
nodejs.ensure
|
|
||||||
cd "${theme}"
|
|
||||||
{
|
|
||||||
npm install
|
|
||||||
npm run watch
|
|
||||||
} 2>&1 \
|
|
||||||
| prefix_stdout "${_Blue}THEME ${1} ${_creset} " \
|
|
||||||
| grep -E --ignore-case --color 'error[s]?[:]? |warning[s]?[:]? |'
|
|
||||||
}
|
|
||||||
|
|
||||||
themes.simple() {
|
|
||||||
( set -e
|
|
||||||
build_msg GRUNT "theme: simple"
|
|
||||||
npm --prefix searx/static/themes/simple run build
|
|
||||||
)
|
|
||||||
dump_return $?
|
|
||||||
}
|
|
||||||
|
|
||||||
themes.simple.test() {
|
|
||||||
build_msg TEST "theme: simple"
|
|
||||||
nodejs.ensure
|
|
||||||
npm --prefix searx/static/themes/simple install
|
|
||||||
npm --prefix searx/static/themes/simple run test
|
|
||||||
dump_return $?
|
|
||||||
}
|
|
||||||
|
|
||||||
PYLINT_FILES=()
|
PYLINT_FILES=()
|
||||||
while IFS= read -r line; do
|
while IFS= read -r line; do
|
||||||
|
|
|
||||||
|
|
@ -20,9 +20,6 @@
|
||||||
# shellcheck source=utils/lib.sh
|
# shellcheck source=utils/lib.sh
|
||||||
. /dev/null
|
. /dev/null
|
||||||
|
|
||||||
# shellcheck disable=SC2034
|
|
||||||
declare main_cmd
|
|
||||||
|
|
||||||
# configure golang environment
|
# configure golang environment
|
||||||
# ----------------------------
|
# ----------------------------
|
||||||
|
|
||||||
|
|
|
||||||
62
utils/lib_sxng_data.sh
Executable file
62
utils/lib_sxng_data.sh
Executable file
|
|
@ -0,0 +1,62 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
data.help(){
|
||||||
|
cat <<EOF
|
||||||
|
data.:
|
||||||
|
all : update searx/sxng_locales.py and searx/data/*
|
||||||
|
traits : update searx/data/engine_traits.json & searx/sxng_locales.py
|
||||||
|
useragents: update searx/data/useragents.json with the most recent versions of Firefox
|
||||||
|
EOF
|
||||||
|
}
|
||||||
|
|
||||||
|
data.all() {
|
||||||
|
( set -e
|
||||||
|
|
||||||
|
pyenv.activate
|
||||||
|
data.traits
|
||||||
|
data.useragents
|
||||||
|
|
||||||
|
build_msg DATA "update searx/data/osm_keys_tags.json"
|
||||||
|
pyenv.cmd python searxng_extra/update/update_osm_keys_tags.py
|
||||||
|
build_msg DATA "update searx/data/ahmia_blacklist.txt"
|
||||||
|
python searxng_extra/update/update_ahmia_blacklist.py
|
||||||
|
build_msg DATA "update searx/data/wikidata_units.json"
|
||||||
|
python searxng_extra/update/update_wikidata_units.py
|
||||||
|
build_msg DATA "update searx/data/currencies.json"
|
||||||
|
python searxng_extra/update/update_currencies.py
|
||||||
|
build_msg DATA "update searx/data/external_bangs.json"
|
||||||
|
python searxng_extra/update/update_external_bangs.py
|
||||||
|
build_msg DATA "update searx/data/engine_descriptions.json"
|
||||||
|
python searxng_extra/update/update_engine_descriptions.py
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
data.traits() {
|
||||||
|
( set -e
|
||||||
|
pyenv.activate
|
||||||
|
build_msg DATA "update searx/data/engine_traits.json"
|
||||||
|
python searxng_extra/update/update_engine_traits.py
|
||||||
|
build_msg ENGINES "update searx/sxng_locales.py"
|
||||||
|
)
|
||||||
|
dump_return $?
|
||||||
|
}
|
||||||
|
|
||||||
|
data.useragents() {
|
||||||
|
build_msg DATA "update searx/data/useragents.json"
|
||||||
|
pyenv.cmd python searxng_extra/update/update_firefox_version.py
|
||||||
|
dump_return $?
|
||||||
|
}
|
||||||
|
|
||||||
|
docs.prebuild() {
|
||||||
|
build_msg DOCS "build ${DOCS_BUILD}/includes"
|
||||||
|
(
|
||||||
|
set -e
|
||||||
|
[ "$VERBOSE" = "1" ] && set -x
|
||||||
|
mkdir -p "${DOCS_BUILD}/includes"
|
||||||
|
./utils/searxng.sh searxng.doc.rst > "${DOCS_BUILD}/includes/searxng.rst"
|
||||||
|
pyenv.cmd searxng_extra/docs_prebuild
|
||||||
|
)
|
||||||
|
dump_return $?
|
||||||
|
}
|
||||||
51
utils/lib_sxng_node.sh
Executable file
51
utils/lib_sxng_node.sh
Executable file
|
|
@ -0,0 +1,51 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
export NODE_MINIMUM_VERSION="16.13.0"
|
||||||
|
|
||||||
|
node.help(){
|
||||||
|
cat <<EOF
|
||||||
|
node.:
|
||||||
|
env : download & install SearXNG's npm dependencies locally
|
||||||
|
env.dev : download & install developer and CI tools
|
||||||
|
clean : drop locally npm installations
|
||||||
|
EOF
|
||||||
|
}
|
||||||
|
|
||||||
|
nodejs.ensure() {
|
||||||
|
if ! nvm.min_node "${NODE_MINIMUM_VERSION}"; then
|
||||||
|
info_msg "install Node.js by NVM"
|
||||||
|
nvm.nodejs
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
node.env() {
|
||||||
|
nodejs.ensure
|
||||||
|
( set -e
|
||||||
|
build_msg INSTALL "./searx/static/themes/simple/package.json"
|
||||||
|
npm --prefix searx/static/themes/simple install
|
||||||
|
)
|
||||||
|
dump_return $?
|
||||||
|
}
|
||||||
|
|
||||||
|
node.env.dev() {
|
||||||
|
nodejs.ensure
|
||||||
|
build_msg INSTALL "./package.json: developer and CI tools"
|
||||||
|
npm install
|
||||||
|
}
|
||||||
|
|
||||||
|
node.clean() {
|
||||||
|
if ! required_commands npm 2>/dev/null; then
|
||||||
|
build_msg CLEAN "npm is not installed / ignore npm dependencies"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
build_msg CLEAN "themes -- locally installed npm dependencies"
|
||||||
|
( set -e
|
||||||
|
npm --prefix searx/static/themes/simple run clean
|
||||||
|
)
|
||||||
|
build_msg CLEAN "locally installed developer and CI tools"
|
||||||
|
( set -e
|
||||||
|
npm --prefix . run clean
|
||||||
|
)
|
||||||
|
dump_return $?
|
||||||
|
}
|
||||||
|
|
@ -12,7 +12,7 @@ STATIC_BUILT_PATHS=(
|
||||||
'searx/templates/simple/icons.html'
|
'searx/templates/simple/icons.html'
|
||||||
)
|
)
|
||||||
|
|
||||||
static_help(){
|
static.help(){
|
||||||
cat <<EOF
|
cat <<EOF
|
||||||
static.build.: ${STATIC_BUILD_COMMIT}
|
static.build.: ${STATIC_BUILD_COMMIT}
|
||||||
commit : build & commit /static folder
|
commit : build & commit /static folder
|
||||||
111
utils/lib_sxng_test.sh
Normal file
111
utils/lib_sxng_test.sh
Normal file
|
|
@ -0,0 +1,111 @@
|
||||||
|
test.help(){
|
||||||
|
cat <<EOF
|
||||||
|
test.:
|
||||||
|
yamllint : lint YAML files (YAMLLINT_FILES)
|
||||||
|
pylint : lint PYLINT_FILES, searx/engines, searx & tests
|
||||||
|
pyright : static type check of python sources
|
||||||
|
black : check black code format
|
||||||
|
unit : run unit tests
|
||||||
|
coverage : run unit tests with coverage
|
||||||
|
robot : run robot test
|
||||||
|
rst : test .rst files incl. README.rst
|
||||||
|
clean : clean intermediate test stuff
|
||||||
|
EOF
|
||||||
|
}
|
||||||
|
|
||||||
|
test.yamllint() {
|
||||||
|
build_msg TEST "[yamllint] \$YAMLLINT_FILES"
|
||||||
|
pyenv.cmd yamllint --strict --format parsable "${YAMLLINT_FILES[@]}"
|
||||||
|
dump_return $?
|
||||||
|
}
|
||||||
|
|
||||||
|
test.pylint() {
|
||||||
|
# shellcheck disable=SC2086
|
||||||
|
( set -e
|
||||||
|
build_msg TEST "[pylint] \$PYLINT_FILES"
|
||||||
|
pyenv.activate
|
||||||
|
python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \
|
||||||
|
--additional-builtins="${PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES}" \
|
||||||
|
"${PYLINT_FILES[@]}"
|
||||||
|
|
||||||
|
build_msg TEST "[pylint] searx/engines"
|
||||||
|
python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \
|
||||||
|
--disable="${PYLINT_SEARXNG_DISABLE_OPTION}" \
|
||||||
|
--additional-builtins="${PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES}" \
|
||||||
|
searx/engines
|
||||||
|
|
||||||
|
build_msg TEST "[pylint] searx tests"
|
||||||
|
python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \
|
||||||
|
--disable="${PYLINT_SEARXNG_DISABLE_OPTION}" \
|
||||||
|
--ignore=searx/engines \
|
||||||
|
searx tests
|
||||||
|
)
|
||||||
|
dump_return $?
|
||||||
|
}
|
||||||
|
|
||||||
|
test.pyright() {
|
||||||
|
build_msg TEST "[pyright] static type check of python sources"
|
||||||
|
node.env.dev
|
||||||
|
# We run Pyright in the virtual environment because Pyright
|
||||||
|
# executes "python" to determine the Python version.
|
||||||
|
build_msg TEST "[pyright] suppress warnings related to intentional monkey patching"
|
||||||
|
pyenv.cmd npx --no-install pyright -p pyrightconfig-ci.json \
|
||||||
|
| grep -v ".py$" \
|
||||||
|
| grep -v '/engines/.*.py.* - warning: "logger" is not defined'\
|
||||||
|
| grep -v '/plugins/.*.py.* - error: "logger" is not defined'\
|
||||||
|
| grep -v '/engines/.*.py.* - warning: "supported_languages" is not defined' \
|
||||||
|
| grep -v '/engines/.*.py.* - warning: "language_aliases" is not defined' \
|
||||||
|
| grep -v '/engines/.*.py.* - warning: "categories" is not defined'
|
||||||
|
dump_return $?
|
||||||
|
}
|
||||||
|
|
||||||
|
test.black() {
|
||||||
|
build_msg TEST "[black] \$BLACK_TARGETS"
|
||||||
|
pyenv.cmd black --check --diff "${BLACK_OPTIONS[@]}" "${BLACK_TARGETS[@]}"
|
||||||
|
dump_return $?
|
||||||
|
}
|
||||||
|
|
||||||
|
test.unit() {
|
||||||
|
build_msg TEST 'tests/unit'
|
||||||
|
pyenv.cmd python -m nose2 -s tests/unit
|
||||||
|
dump_return $?
|
||||||
|
}
|
||||||
|
|
||||||
|
test.coverage() {
|
||||||
|
build_msg TEST 'unit test coverage'
|
||||||
|
( set -e
|
||||||
|
pyenv.activate
|
||||||
|
python -m nose2 -C --log-capture --with-coverage --coverage searx -s tests/unit
|
||||||
|
coverage report
|
||||||
|
coverage html
|
||||||
|
)
|
||||||
|
dump_return $?
|
||||||
|
}
|
||||||
|
|
||||||
|
test.robot() {
|
||||||
|
build_msg TEST 'robot'
|
||||||
|
gecko.driver
|
||||||
|
PYTHONPATH=. pyenv.cmd python -m tests.robot
|
||||||
|
dump_return $?
|
||||||
|
}
|
||||||
|
|
||||||
|
test.rst() {
|
||||||
|
build_msg TEST "[reST markup] ${RST_FILES[*]}"
|
||||||
|
for rst in "${RST_FILES[@]}"; do
|
||||||
|
pyenv.cmd rst2html.py --halt error "$rst" > /dev/null || die 42 "fix issue in $rst"
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
test.pybabel() {
|
||||||
|
TEST_BABEL_FOLDER="build/test/pybabel"
|
||||||
|
build_msg TEST "[extract messages] pybabel"
|
||||||
|
mkdir -p "${TEST_BABEL_FOLDER}"
|
||||||
|
pyenv.cmd pybabel extract -F babel.cfg -o "${TEST_BABEL_FOLDER}/messages.pot" searx
|
||||||
|
}
|
||||||
|
|
||||||
|
test.clean() {
|
||||||
|
build_msg CLEAN "test stuff"
|
||||||
|
rm -rf geckodriver.log .coverage coverage/
|
||||||
|
dump_return $?
|
||||||
|
}
|
||||||
|
|
||||||
65
utils/lib_sxng_themes.sh
Executable file
65
utils/lib_sxng_themes.sh
Executable file
|
|
@ -0,0 +1,65 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
declare _Blue
|
||||||
|
declare _creset
|
||||||
|
|
||||||
|
themes.help(){
|
||||||
|
cat <<EOF
|
||||||
|
themes.:
|
||||||
|
all : build all themes
|
||||||
|
live : to get live builds of CSS & JS use 'LIVE_THEME=simple make run'
|
||||||
|
simple.:
|
||||||
|
build : build simple theme
|
||||||
|
test : test simple theme
|
||||||
|
EOF
|
||||||
|
}
|
||||||
|
|
||||||
|
themes.all() {
|
||||||
|
( set -e
|
||||||
|
pygments.less
|
||||||
|
node.env
|
||||||
|
themes.simple
|
||||||
|
)
|
||||||
|
dump_return $?
|
||||||
|
}
|
||||||
|
|
||||||
|
themes.live() {
|
||||||
|
local LIVE_THEME="${LIVE_THEME:-${1}}"
|
||||||
|
case "${LIVE_THEME}" in
|
||||||
|
simple)
|
||||||
|
theme="searx/static/themes/${LIVE_THEME}"
|
||||||
|
;;
|
||||||
|
'')
|
||||||
|
die_caller 42 "missing theme argument"
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
die_caller 42 "unknown theme '${LIVE_THEME}' // [simple]'"
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
build_msg GRUNT "theme: $1 (live build)"
|
||||||
|
nodejs.ensure
|
||||||
|
cd "${theme}"
|
||||||
|
{
|
||||||
|
npm install
|
||||||
|
npm run watch
|
||||||
|
} 2>&1 \
|
||||||
|
| prefix_stdout "${_Blue}THEME ${1} ${_creset} " \
|
||||||
|
| grep -E --ignore-case --color 'error[s]?[:]? |warning[s]?[:]? |'
|
||||||
|
}
|
||||||
|
|
||||||
|
themes.simple() {
|
||||||
|
( set -e
|
||||||
|
build_msg GRUNT "theme: simple"
|
||||||
|
npm --prefix searx/static/themes/simple run build
|
||||||
|
)
|
||||||
|
dump_return $?
|
||||||
|
}
|
||||||
|
|
||||||
|
themes.simple.test() {
|
||||||
|
build_msg TEST "theme: simple"
|
||||||
|
nodejs.ensure
|
||||||
|
npm --prefix searx/static/themes/simple install
|
||||||
|
npm --prefix searx/static/themes/simple run test
|
||||||
|
dump_return $?
|
||||||
|
}
|
||||||
211
utils/lib_sxng_weblate.sh
Executable file
211
utils/lib_sxng_weblate.sh
Executable file
|
|
@ -0,0 +1,211 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
weblate.help(){
|
||||||
|
cat <<EOF
|
||||||
|
weblate.:
|
||||||
|
push.translations: push translation changes from SearXNG to Weblate's counterpart
|
||||||
|
to.translations: Update 'translations' branch with last additions from Weblate.
|
||||||
|
EOF
|
||||||
|
}
|
||||||
|
|
||||||
|
TRANSLATIONS_WORKTREE="$CACHE/translations"
|
||||||
|
|
||||||
|
weblate.translations.worktree() {
|
||||||
|
|
||||||
|
# Create git worktree ${TRANSLATIONS_WORKTREE} and checkout branch
|
||||||
|
# 'translations' from Weblate's counterpart (weblate) of the SearXNG
|
||||||
|
# (origin).
|
||||||
|
#
|
||||||
|
# remote weblate https://translate.codeberg.org/git/searxng/searxng/
|
||||||
|
|
||||||
|
( set -e
|
||||||
|
if ! git remote get-url weblate 2> /dev/null; then
|
||||||
|
git remote add weblate https://translate.codeberg.org/git/searxng/searxng/
|
||||||
|
fi
|
||||||
|
if [ -d "${TRANSLATIONS_WORKTREE}" ]; then
|
||||||
|
pushd .
|
||||||
|
cd "${TRANSLATIONS_WORKTREE}"
|
||||||
|
git reset --hard HEAD
|
||||||
|
git pull origin translations
|
||||||
|
popd
|
||||||
|
else
|
||||||
|
mkdir -p "${TRANSLATIONS_WORKTREE}"
|
||||||
|
git worktree add "${TRANSLATIONS_WORKTREE}" translations
|
||||||
|
fi
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
weblate.to.translations() {
|
||||||
|
|
||||||
|
# Update 'translations' branch of SearXNG (origin) with last additions from
|
||||||
|
# Weblate.
|
||||||
|
|
||||||
|
# 1. Check if Weblate is locked, if not die with error message
|
||||||
|
# 2. On Weblate's counterpart (weblate), pull master and translations branch
|
||||||
|
# from SearXNG (origin).
|
||||||
|
# 3. Commit changes made in a Weblate object on Weblate's counterpart
|
||||||
|
# (weblate).
|
||||||
|
# 4. In translations worktree, merge changes of branch 'translations' from
|
||||||
|
# remote 'weblate' and push it on branch 'translations' of 'origin'
|
||||||
|
|
||||||
|
( set -e
|
||||||
|
pyenv.activate
|
||||||
|
if [ "$(wlc lock-status)" != "locked: True" ]; then
|
||||||
|
die 1 "weblate must be locked, currently: $(wlc lock-status)"
|
||||||
|
fi
|
||||||
|
# weblate: commit pending changes
|
||||||
|
wlc pull
|
||||||
|
wlc commit
|
||||||
|
|
||||||
|
# get the translations in a worktree
|
||||||
|
weblate.translations.worktree
|
||||||
|
|
||||||
|
pushd "${TRANSLATIONS_WORKTREE}"
|
||||||
|
git remote update weblate
|
||||||
|
git merge weblate/translations
|
||||||
|
git push
|
||||||
|
popd
|
||||||
|
)
|
||||||
|
dump_return $?
|
||||||
|
}
|
||||||
|
|
||||||
|
weblate.translations.commit() {
|
||||||
|
|
||||||
|
# Update 'translations' branch of SearXNG (origin) with last additions from
|
||||||
|
# Weblate. Copy the changes to the master branch, compile translations and
|
||||||
|
# create a commit in the local branch (master)
|
||||||
|
|
||||||
|
local existing_commit_hash commit_body commit_message exitcode
|
||||||
|
( set -e
|
||||||
|
pyenv.activate
|
||||||
|
# lock change on weblate
|
||||||
|
wlc lock
|
||||||
|
|
||||||
|
# get translations branch in git worktree (TRANSLATIONS_WORKTREE)
|
||||||
|
weblate.translations.worktree
|
||||||
|
existing_commit_hash=$(cd "${TRANSLATIONS_WORKTREE}"; git log -n1 --pretty=format:'%h')
|
||||||
|
|
||||||
|
# pull weblate commits
|
||||||
|
weblate.to.translations
|
||||||
|
|
||||||
|
# copy the changes to the master branch
|
||||||
|
cp -rv --preserve=mode,timestamps "${TRANSLATIONS_WORKTREE}/searx/translations" "searx"
|
||||||
|
|
||||||
|
# compile translations
|
||||||
|
build_msg BABEL 'compile translation catalogs into binary MO files'
|
||||||
|
pybabel compile --statistics \
|
||||||
|
-d "searx/translations"
|
||||||
|
# git add/commit (no push)
|
||||||
|
commit_body=$(cd "${TRANSLATIONS_WORKTREE}"; git log --pretty=format:'%h - %as - %aN <%ae>' "${existing_commit_hash}..HEAD")
|
||||||
|
commit_message=$(echo -e "[translations] update from Weblate\n\n${commit_body}")
|
||||||
|
git add searx/translations
|
||||||
|
git commit -m "${commit_message}"
|
||||||
|
)
|
||||||
|
exitcode=$?
|
||||||
|
( # make sure to always unlock weblate
|
||||||
|
set -e
|
||||||
|
pyenv.cmd wlc unlock
|
||||||
|
)
|
||||||
|
dump_return $exitcode
|
||||||
|
}
|
||||||
|
|
||||||
|
weblate.push.translations() {
|
||||||
|
|
||||||
|
# Push *translation changes* from SearXNG (origin) to Weblate's counterpart
|
||||||
|
# (weblate).
|
||||||
|
|
||||||
|
# In branch master of SearXNG (origin) check for meaningful changes in
|
||||||
|
# folder 'searx/translations', commit changes on branch 'translations' and
|
||||||
|
# at least, pull updated branches on Weblate's counterpart (weblate).
|
||||||
|
|
||||||
|
# 1. Create git worktree ${TRANSLATIONS_WORKTREE} and checkout branch
|
||||||
|
# 'translations' from remote 'weblate'.
|
||||||
|
# 2. Stop if there is no meaningful change in the 'master' branch (origin),
|
||||||
|
# compared to the 'translations' branch (weblate), otherwise ...
|
||||||
|
# 3. Update 'translations' branch of SearXNG (origin) with last additions
|
||||||
|
# from Weblate.
|
||||||
|
# 5. Notify Weblate to pull updated 'master' & 'translations' branch.
|
||||||
|
|
||||||
|
local messages_pot diff_messages_pot last_commit_hash last_commit_detail \
|
||||||
|
exitcode
|
||||||
|
messages_pot="${TRANSLATIONS_WORKTREE}/searx/translations/messages.pot"
|
||||||
|
( set -e
|
||||||
|
pyenv.activate
|
||||||
|
# get translations branch in git worktree (TRANSLATIONS_WORKTREE)
|
||||||
|
weblate.translations.worktree
|
||||||
|
|
||||||
|
# update messages.pot in the master branch
|
||||||
|
build_msg BABEL 'extract messages from source files and generate POT file'
|
||||||
|
pybabel extract -F babel.cfg \
|
||||||
|
-o "${messages_pot}" \
|
||||||
|
"searx/"
|
||||||
|
|
||||||
|
# stop if there is no meaningful change in the master branch
|
||||||
|
diff_messages_pot=$(cd "${TRANSLATIONS_WORKTREE}";\
|
||||||
|
git diff -- "searx/translations/messages.pot")
|
||||||
|
if ! echo "$diff_messages_pot" | grep -qE "[\+\-](msgid|msgstr)"; then
|
||||||
|
build_msg BABEL 'no changes detected, exiting'
|
||||||
|
return 42
|
||||||
|
fi
|
||||||
|
return 0
|
||||||
|
)
|
||||||
|
exitcode=$?
|
||||||
|
if [ "$exitcode" -eq 42 ]; then
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
if [ "$exitcode" -gt 0 ]; then
|
||||||
|
return $exitcode
|
||||||
|
fi
|
||||||
|
(
|
||||||
|
set -e
|
||||||
|
pyenv.activate
|
||||||
|
|
||||||
|
# lock change on weblate
|
||||||
|
# weblate may add commit(s) since the call to "weblate.translations.worktree".
|
||||||
|
# this is not a problem because after this line, "weblate.to.translations"
|
||||||
|
# calls again "weblate.translations.worktree" which calls "git pull"
|
||||||
|
wlc lock
|
||||||
|
|
||||||
|
# save messages.pot in the translations branch for later
|
||||||
|
pushd "${TRANSLATIONS_WORKTREE}"
|
||||||
|
git stash push
|
||||||
|
popd
|
||||||
|
|
||||||
|
# merge weblate commits into the translations branch
|
||||||
|
weblate.to.translations
|
||||||
|
|
||||||
|
# restore messages.pot in the translations branch
|
||||||
|
pushd "${TRANSLATIONS_WORKTREE}"
|
||||||
|
git stash pop
|
||||||
|
popd
|
||||||
|
|
||||||
|
# update messages.po files in the master branch
|
||||||
|
build_msg BABEL 'update existing message catalogs from POT file'
|
||||||
|
pybabel update -N \
|
||||||
|
-i "${messages_pot}" \
|
||||||
|
-d "${TRANSLATIONS_WORKTREE}/searx/translations"
|
||||||
|
|
||||||
|
# git add/commit/push
|
||||||
|
last_commit_hash=$(git log -n1 --pretty=format:'%h')
|
||||||
|
last_commit_detail=$(git log -n1 --pretty=format:'%h - %as - %aN <%ae>' "${last_commit_hash}")
|
||||||
|
|
||||||
|
pushd "${TRANSLATIONS_WORKTREE}"
|
||||||
|
git add searx/translations
|
||||||
|
git commit \
|
||||||
|
-m "[translations] update messages.pot and messages.po files" \
|
||||||
|
-m "From ${last_commit_detail}"
|
||||||
|
git push
|
||||||
|
popd
|
||||||
|
|
||||||
|
# notify weblate to pull updated master & translations branch
|
||||||
|
wlc pull
|
||||||
|
)
|
||||||
|
exitcode=$?
|
||||||
|
( # make sure to always unlock weblate
|
||||||
|
set -e
|
||||||
|
pyenv.activate
|
||||||
|
wlc unlock
|
||||||
|
)
|
||||||
|
dump_return $exitcode
|
||||||
|
}
|
||||||
Loading…
Reference in a new issue