From fe4a33a2bf1b11cd0b35dfdb1f3ad84929fffe06 Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Fri, 25 Feb 2022 22:43:05 +0100 Subject: [PATCH 1/5] Replace morty & filtron by redis & the limiter plugin --- .env | 2 +- Caddyfile | 38 +++++++++++++---------------- docker-compose.yaml | 57 +++++++++++++++----------------------------- searxng/settings.yml | 6 +++++ 4 files changed, 43 insertions(+), 60 deletions(-) create mode 100644 searxng/settings.yml diff --git a/.env b/.env index befe5bd..bae61d2 100644 --- a/.env +++ b/.env @@ -11,4 +11,4 @@ SEARXNG_COMMAND=-f # use openssl rand -base64 33 -MORTY_KEY=ReplaceWithARealKey! +SEARXNG_SECRET=ultrasecretkey diff --git a/Caddyfile b/Caddyfile index 299e769..ab36456 100644 --- a/Caddyfile +++ b/Caddyfile @@ -11,25 +11,26 @@ @api { path /config - path /status + path /status } @static { path /static/* } + @imageproxy { + path /image_proxy + } + + @notimageproxy { + not path /image_proxy + } + + @notstatic { not path /static/* } - @morty { - path /morty/* - } - - @notmorty { - not path /morty/* - } - header { # Enable HTTP Strict Transport Security (HSTS) to force clients to always connect via HTTPS Strict-Transport-Security "max-age=31536000; includeSubDomains; preload" @@ -67,8 +68,8 @@ # Cache header @static { # Cache - Cache-Control "public, max-age=31536000" - defer + Cache-Control "public, max-age=31536000" + defer } header @notstatic { @@ -78,24 +79,19 @@ } # CSP (see http://content-security-policy.com/ ) - header @morty { - Content-Security-Policy "default-src 'none'; style-src 'self' 'unsafe-inline'; form-action 'self'; frame-ancestors 'self'; base-uri 'self'; img-src 'self' data:; font-src 'self'; frame-src 'self'" + header @imageproxy { + Content-Security-Policy "default-src 'none'; img-src 'self' data:" } - header @notmorty { + header @notimageproxy { Content-Security-Policy "upgrade-insecure-requests; default-src 'none'; script-src 'self'; style-src 'self' 'unsafe-inline'; form-action 'self'; font-src 'self'; frame-ancestors 'self'; base-uri 'self'; connect-src 'self' https://overpass-api.de; img-src 'self' data: https://*.tile.openstreetmap.org; frame-src https://www.youtube-nocookie.com https://player.vimeo.com https://www.dailymotion.com https://www.deezer.com https://www.mixcloud.com https://w.soundcloud.com https://embed.spotify.com" } - # Morty - handle @morty { - reverse_proxy localhost:3000 - } - - # Filtron + # Searx handle { encode zstd gzip - reverse_proxy localhost:4040 { + reverse_proxy localhost:8080 { header_up X-Forwarded-Port {http.request.port} header_up X-Forwarded-Proto {http.request.scheme} header_up X-Forwarded-TlsProto {tls_protocol} diff --git a/docker-compose.yaml b/docker-compose.yaml index 46a77bb..6fe09cb 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -5,14 +5,13 @@ services: caddy: container_name: caddy image: caddy:2-alpine - restart: on-failure network_mode: host volumes: - ./Caddyfile:/etc/caddy/Caddyfile:ro - caddy-data:/data:rw - caddy-config:/config:rw environment: - - SEARXNG_HOSTNAME=${SEARXNG_HOSTNAME:-localhost} + - SEARXNG_HOSTNAME=${SEARXNG_HOSTNAME:-http://localhost:80} - SEARXNG_TLS=${LETSENCRYPT_EMAIL:-internal} cap_drop: - ALL @@ -20,36 +19,36 @@ services: - NET_BIND_SERVICE - DAC_OVERRIDE - filtron: - container_name: filtron - image: dalf/filtron - restart: always - ports: - - "127.0.0.1:4040:4040" - - "127.0.0.1:4041:4041" + redis: + container_name: redis + image: "redis:alpine" + command: redis-server --save "" --appendonly "no" networks: - searxng - command: -listen 0.0.0.0:4040 -api 0.0.0.0:4041 -target searxng:8080 - volumes: - - ./rules.json:/etc/filtron/rules.json:rw - read_only: true + tmpfs: + - /var/lib/redis cap_drop: - ALL + cap_add: + - SETGID + - SETUID + - DAC_OVERRIDE - searxng: - container_name: searxng - image: searxng/searxng:latest + searx: + container_name: searx + image: searxng/searxng restart: always networks: - searxng - command: ${SEARXNG_COMMAND:-} + command: ${SEARX_COMMAND:-} + ports: + - "127.0.0.1:8080:8080" volumes: - - ./searxng:/etc/searxng:rw + - ./searxng:/etc/searx:rw environment: - BIND_ADDRESS=0.0.0.0:8080 - BASE_URL=https://${SEARXNG_HOSTNAME:-localhost}/ - - MORTY_URL=https://${SEARXNG_HOSTNAME:-localhost}/morty/ - - MORTY_KEY=${MORTY_KEY} + - SEARXNG_SECRET=${SEARXNG_SECRET} cap_drop: - ALL cap_add: @@ -58,24 +57,6 @@ services: - SETUID - DAC_OVERRIDE - morty: - container_name: morty - image: dalf/morty - restart: always - ports: - - "127.0.0.1:3000:3000" - networks: - - searxng - command: -timeout 6 -ipv6 - environment: - - MORTY_KEY=${MORTY_KEY} - - MORTY_ADDRESS=0.0.0.0:3000 - logging: - driver: none - read_only: true - cap_drop: - - ALL - networks: searxng: ipam: diff --git a/searxng/settings.yml b/searxng/settings.yml new file mode 100644 index 0000000..0c087ad --- /dev/null +++ b/searxng/settings.yml @@ -0,0 +1,6 @@ +use_default_settings: true +server: + limiter: true + image_proxy: true +redis: + url: redis://redis:6379/0 From 8a4db4caa0951794a27908e7595b868593d6798a Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Fri, 25 Feb 2022 23:10:07 +0100 Subject: [PATCH 2/5] Update README.md --- README.md | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 6382096..07a2704 100644 --- a/README.md +++ b/README.md @@ -7,21 +7,20 @@ Create a new SearXNG instance in five minutes using Docker | Name | Description | Docker image | Dockerfile | | -- | -- | -- | -- | | [Caddy](https://github.com/caddyserver/caddy) | Reverse proxy (create a LetsEncrypt certificate automatically) | [caddy/caddy:2-alpine](https://hub.docker.com/_/caddy) | [Dockerfile](https://github.com/caddyserver/caddy-docker) | -| [Filtron](https://github.com/dalf/filtron) | Filtering reverse HTTP proxy, bot and abuse protection | [dalf/filtron:latest](https://hub.docker.com/r/dalf/filtron) | See [asciimoo/filtron#4](https://github.com/asciimoo/filtron/pull/4) | | [SearXNG](https://github.com/searxng/searxng) | SearXNG by itself | [searxng/searxng:latest](https://hub.docker.com/r/searxng/searxng) | [Dockerfile](https://github.com/searxng/searxng/blob/master/Dockerfile) | -| [Morty](https://github.com/dalf/morty) | Privacy aware web content sanitizer proxy as a service. | [dalf/morty:latest](https://hub.docker.com/r/dalf/morty) | [Dockerfile](https://github.com/dalf/morty/blob/master/Dockerfile) | +| [Redis](https://github.com/redis/redis) | In-memory database | [redis:alpine](https://hub.docker.com/_/redis) | [Dockerfile-alpine.template](https://github.com/docker-library/redis/blob/master/Dockerfile-alpine.template) | ## How to use it - [Install docker](https://docs.docker.com/install/) - [Install docker-compose](https://docs.docker.com/compose/install/) (be sure that docker-compose version is at least 1.9.0) - only on MacOSX: ```brew install coreutils``` to install ```greadlink``` -- Get searxng-docker: +- Get searxng-docker ```sh cd /usr/local git clone https://github.com/searxng/searxng-docker.git cd searxng-docker ``` -- Generate MORTY_KEY ```sed -i "s|ReplaceWithARealKey\!|$(openssl rand -base64 33)|g" .env``` +- Generate the secret key ```sed -i "s|ultrasecretkey|$(openssl rand -hex 32)|g" .env``` - Edit the other settings in [.env](https://github.com/searxng/searxng-docker/blob/master/.env) file according to your need - Check everything is working: ```./start.sh``` - ```cp searxng-docker.service.template searxng-docker.service``` @@ -34,9 +33,9 @@ Create a new SearXNG instance in five minutes using Docker ## Note on the image proxy feature -The SearXNG image proxy is activated by default using [Morty](https://github.com/dalf/morty). +The SearXNG image proxy is activated by default. -The default [Content-Security-Policy](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Security-Policy) allow the browser to access to {SEARXNG_HOSTNAME} and ```https://*.tile.openstreetmap.org;```. +The default [Content-Security-Policy](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Security-Policy) allow the browser to access to ```${SEARXNG_HOSTNAME}``` and ```https://*.tile.openstreetmap.org;```. If some users wants to disable the image proxy, you have to modify [./Caddyfile](https://github.com/searxng/searxng-docker/blob/master/Caddyfile). Replace the ```img-src 'self' data: https://*.tile.openstreetmap.org;``` by ```img-src * data:;```. @@ -52,17 +51,11 @@ It is possible to use the [extend feature](https://docs.docker.com/compose/exten ## Multi Architecture Docker images -For now only the amd64 platform is supported. +Supported architecture: +- amd64 +- arm64 +- arm/v7 ## How to update ? -Check the content of [```update.sh```](https://github.com/searxng/searxng-docker/blob/master/update.sh). - -## Access to the Filtron API - -The [Filtron API](https://github.com/dalf/filtron#api) is available on ```http://localhost:4041```. - -For example, to display the loaded rules: -``` -curl http://localhost:4041/rules | jq -``` +Check the content of [```update.sh```](https://github.com/searxng/searxng-docker/blob/master/update.sh) From 8952fad37eb7e49f0b93bfe3a69c43ac6a7ccfcc Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Fri, 25 Feb 2022 23:37:40 +0100 Subject: [PATCH 3/5] Remove rules.json --- rules.json | 147 ----------------------------------------------------- 1 file changed, 147 deletions(-) delete mode 100644 rules.json diff --git a/rules.json b/rules.json deleted file mode 100644 index e77e93a..0000000 --- a/rules.json +++ /dev/null @@ -1,147 +0,0 @@ -[ - { - "name": "searx.space", - "filters": ["Header:X-Forwarded-For=nslookup(check.searx.space)"], - "stop": true, - "actions": [{ "name": "log"}] - }, - { - "name": "IP limit, all paths", - "interval": 3, - "limit": 25, - "aggregations": ["Header:X-Forwarded-For"], - "actions": [ - {"name": "block", - "params": {"message": "Rate limit exceeded, try again later."}} - ] - }, - { - "name": "useragent limit, all paths", - "interval": 30, - "limit": 200, - "aggregations": ["Header:X-Forwarded-For", "Header:User-Agent"], - "stop": true, - "actions": [ - {"name": "block", - "params": {"message": "Rate limit exceeded, try again later."}} - ] - }, - { - "name": "search request", - "filters": ["Param:q", "Path=^(/|/search)$"], - "subrules": [ - { - "name": "allow Firefox Android (issue #48 and #60)", - "filters": [ - "Param:q=^1$", - "Header:User-Agent=(^MozacFetch/[0-9]{2,3}.[0-9].[0-9]+$|^Mozilla/5.0 \\(Android [0-9]{1,2}(.[0-9]{1,2}.[0-9]{1,2})?; Mobile; rv:[0-9]{2,3}.[0-9]\\) Gecko/[0-9]{2,3}.[0-9] Firefox/[0-9]{2,3}.[0-9]$)" - ], - "stop": true, - "actions": [{"name": "log"}] - }, - { - "name": "robot agent forbidden", - "limit": 0, - "stop": true, - "filters": ["Header:User-Agent=([Cc][Uu][Rr][Ll]|[wW]get|Scrapy|splash|JavaFX|FeedFetcher|python-requests|Go-http-client|Java|Jakarta|okhttp|HttpClient|Jersey|Python|libwww-perl|Ruby|SynHttpClient|UniversalFeedParser)"], - "actions": [ - {"name": "block", - "params": {"message": "Rate limit exceeded"}} - ] - }, - { - "name": "bot forbidden", - "limit": 0, - "stop": true, - "filters": ["Header:User-Agent=(Googlebot|GoogleImageProxy|bingbot|Baiduspider|yacybot|YandexMobileBot|YandexBot|Yahoo! Slurp|MJ12bot|AhrefsBot|archive.org_bot|msnbot|MJ12bot|SeznamBot|linkdexbot|Netvibes|SMTBot|zgrab|James BOT|Sogou|Abonti|Pixray|Spinn3r|SemrushBot|Exabot|ZmEu|BLEXBot|bitlybot)"], - "actions": [ - {"name": "block", - "params": {"message": "Rate limit exceeded"}} - ] - }, - { - "name": "block missing accept-language", - "filters": ["!Header:Accept-Language"], - "limit": 0, - "stop": true, - "actions": [ - {"name": "block", - "params": {"message": "Rate limit exceeded"}} - ] - }, - { - "name": "block Connection:close", - "filters": ["Header:Connection=close"], - "limit": 0, - "stop": true, - "actions": [ - {"name": "block", - "params": {"message": "Rate limit exceeded"}} - ] - }, - { - "name": "block no gzip support", - "filters": ["!Header:Accept-Encoding=(^gzip$|^gzip[;,]|[; ]gzip$|[; ]gzip[;,])"], - "limit": 0, - "stop": true, - "actions": [ - {"name": "block", - "params": {"message": "Rate limit exceeded"}} - ] - }, - { - "name": "block no deflate support", - "filters": ["!Header:Accept-Encoding=(^deflate$|^deflate[;,]|[; ]deflate$|[; ]deflate[;,])"], - "limit": 0, - "stop": true, - "actions": [ - {"name": "block", - "params": {"message": "Rate limit exceeded"}} - ] - }, - { - "name": "block accept everything", - "filters": ["!Header:Accept=text/html"], - "limit": 0, - "stop": true, - "actions": [ - {"name": "block", - "params": {"message": "Rate limit exceeded"}} - ] - }, - { - "name": "rss/json limit", - "interval": 3600, - "limit": 4, - "stop": true, - "filters": ["Param:format=(csv|json|rss)"], - "aggregations": ["Header:X-Forwarded-For"], - "actions": [ - {"name": "block", - "params": {"message": "Rate limit exceeded, try again later."}} - ] - }, - { - "name": "IP limit", - "interval": 3, - "limit": 3, - "aggregations": ["Header:X-Forwarded-For"], - "actions": [ - {"name": "block", - "params": {"message": "Rate limit exceeded, try again later."}} - ] - }, - { - "name": "IP and useragent limit", - "interval": 600, - "limit": 60, - "stop": true, - "aggregations": ["Header:X-Forwarded-For", "Header:User-Agent"], - "actions": [ - {"name": "block", - "params": {"message": "Rate limit exceeded, try again later."}} - ] - } - ] - } -] From 0a1db38e21815c3ba6332e6ae5de2ce2eb8e7730 Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Sat, 19 Mar 2022 20:16:38 +0100 Subject: [PATCH 4/5] Rely on searxng/settings.yml --- .env | 7 ------- .gitignore | 2 +- Caddyfile | 23 +++++++++-------------- README.md | 5 +++-- docker-compose.yaml | 19 ++++++++++--------- searxng/settings.yml | 5 ++++- 6 files changed, 27 insertions(+), 34 deletions(-) diff --git a/.env b/.env index bae61d2..fc0f666 100644 --- a/.env +++ b/.env @@ -5,10 +5,3 @@ # SEARXNG_HOSTNAME= # LETSENCRYPT_EMAIL= - -# automatically update settings to the new version -# comment this line if you made / will make some modifications to the settings -SEARXNG_COMMAND=-f - -# use openssl rand -base64 33 -SEARXNG_SECRET=ultrasecretkey diff --git a/.gitignore b/.gitignore index f72896e..d44b70f 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,4 @@ searxng-docker.service caddy srv -searx +searxng/uwsgi.ini diff --git a/Caddyfile b/Caddyfile index ab36456..d5729d2 100644 --- a/Caddyfile +++ b/Caddyfile @@ -11,13 +11,19 @@ @api { path /config - path /status + path /healthz + path /stats/errors + path /stats/checker } @static { path /static/* } + @notstatic { + not path /static/* + } + @imageproxy { path /image_proxy } @@ -26,11 +32,6 @@ not path /image_proxy } - - @notstatic { - not path /static/* - } - header { # Enable HTTP Strict Transport Security (HSTS) to force clients to always connect via HTTPS Strict-Transport-Security "max-age=31536000; includeSubDomains; preload" @@ -41,11 +42,8 @@ # Prevent some browsers from MIME-sniffing a response away from the declared Content-Type X-Content-Type-Options "nosniff" - # Disallow the site to be rendered within a frame (clickjacking protection) - X-Frame-Options "SAMEORIGIN" - # Disable some features - Permissions-Policy "accelerometer=();ambient-light-sensor=(); autoplay=();camera=();encrypted-media=();focus-without-user-activation=(); geolocation=();gyroscope=();magnetometer=();microphone=();midi=();payment=();picture-in-picture=(); speaker=();sync-xhr=();usb=();vr=()" + Permissions-Policy "accelerometer=(),ambient-light-sensor=(),autoplay=(),camera=(),encrypted-media=(),focus-without-user-activation=(),geolocation=(),gyroscope=(),magnetometer=(),microphone=(),midi=(),payment=(),picture-in-picture=(),speaker=(),sync-xhr=(),usb=(),vr=()" # Disable some features (legacy) Feature-Policy "accelerometer 'none';ambient-light-sensor 'none'; autoplay 'none';camera 'none';encrypted-media 'none';focus-without-user-activation 'none'; geolocation 'none';gyroscope 'none';magnetometer 'none';microphone 'none';midi 'none';payment 'none';picture-in-picture 'none'; speaker 'none';sync-xhr 'none';usb 'none';vr 'none'" @@ -87,16 +85,13 @@ Content-Security-Policy "upgrade-insecure-requests; default-src 'none'; script-src 'self'; style-src 'self' 'unsafe-inline'; form-action 'self'; font-src 'self'; frame-ancestors 'self'; base-uri 'self'; connect-src 'self' https://overpass-api.de; img-src 'self' data: https://*.tile.openstreetmap.org; frame-src https://www.youtube-nocookie.com https://player.vimeo.com https://www.dailymotion.com https://www.deezer.com https://www.mixcloud.com https://w.soundcloud.com https://embed.spotify.com" } - # Searx + # SearXNG handle { encode zstd gzip reverse_proxy localhost:8080 { header_up X-Forwarded-Port {http.request.port} header_up X-Forwarded-Proto {http.request.scheme} - header_up X-Forwarded-TlsProto {tls_protocol} - header_up X-Forwarded-TlsCipher {tls_cipher} - header_up X-Forwarded-HttpsProto {proto} } } diff --git a/README.md b/README.md index 07a2704..ab6cbcc 100644 --- a/README.md +++ b/README.md @@ -20,8 +20,9 @@ Create a new SearXNG instance in five minutes using Docker git clone https://github.com/searxng/searxng-docker.git cd searxng-docker ``` -- Generate the secret key ```sed -i "s|ultrasecretkey|$(openssl rand -hex 32)|g" .env``` -- Edit the other settings in [.env](https://github.com/searxng/searxng-docker/blob/master/.env) file according to your need +- Edit the [.env](https://github.com/searxng/searxng-docker/blob/master/.env) file to set the hostname and an email +- Generate the secret key ```sed -i "s|ultrasecretkey|$(openssl rand -hex 32)|g" searxng/settings.yml``` +- Edit the [searxng/settings.yml](https://github.com/searxng/searxng-docker/blob/master/searxng/settings.yml) file according to your need - Check everything is working: ```./start.sh``` - ```cp searxng-docker.service.template searxng-docker.service``` - edit the content of ```WorkingDirectory``` in the ```searxng-docker.service``` file (only if the installation path is different from /usr/local/searxng-docker) diff --git a/docker-compose.yaml b/docker-compose.yaml index 6fe09cb..4a153f9 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -34,21 +34,18 @@ services: - SETUID - DAC_OVERRIDE - searx: - container_name: searx - image: searxng/searxng + searxng: + container_name: searxng + image: searxng/searxng:latest restart: always networks: - searxng - command: ${SEARX_COMMAND:-} ports: - "127.0.0.1:8080:8080" volumes: - - ./searxng:/etc/searx:rw + - ./searxng:/etc/searxng:rw environment: - - BIND_ADDRESS=0.0.0.0:8080 - - BASE_URL=https://${SEARXNG_HOSTNAME:-localhost}/ - - SEARXNG_SECRET=${SEARXNG_SECRET} + - SEARXNG_BASE_URL=https://${SEARXNG_HOSTNAME:-localhost}/ cap_drop: - ALL cap_add: @@ -56,7 +53,11 @@ services: - SETGID - SETUID - DAC_OVERRIDE - + logging: + driver: "json-file" + options: + max-size: "1m" + max-file: "1" networks: searxng: ipam: diff --git a/searxng/settings.yml b/searxng/settings.yml index 0c087ad..4be7731 100644 --- a/searxng/settings.yml +++ b/searxng/settings.yml @@ -1,6 +1,9 @@ +# see https://docs.searxng.org/admin/engines/settings.html#use-default-settings use_default_settings: true server: - limiter: true + # base_url is defined in the SEARXNG_BASE_URL environment variable, see .env and docker-compose.yml + secret_key: "ultrasecretkey" # change this! + limiter: true # can be disabled for a private instance image_proxy: true redis: url: redis://redis:6379/0 From 6b0d7c591bab3d16822942b1530971f2246236dd Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Sat, 2 Apr 2022 21:31:05 +0200 Subject: [PATCH 5/5] static files: use hashes --- searxng/settings.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/searxng/settings.yml b/searxng/settings.yml index 4be7731..d0df82c 100644 --- a/searxng/settings.yml +++ b/searxng/settings.yml @@ -5,5 +5,7 @@ server: secret_key: "ultrasecretkey" # change this! limiter: true # can be disabled for a private instance image_proxy: true +ui: + static_use_hash: true redis: url: redis://redis:6379/0