Ponysearch/ponysearch-docker
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #12 from dalf/remove_filtron_morty

Browse source 
Replace morty & filtron by redis & the limiter plugin
This commit is contained in:
Paul Braeuning 2022-04-20 01:25:46 +02:00 committed by GitHub
commit 7f01f0d93d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 57 additions and 233 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
.env
View file

@ -5,10 +5,3 @@
# SEARXNG_HOSTNAME=<host> # SEARXNG_HOSTNAME=<host>
# LETSENCRYPT_EMAIL=<email> # LETSENCRYPT_EMAIL=<email>
# automatically update settings to the new version
# comment this line if you made / will make some modifications to the settings
SEARXNG_COMMAND=-f
# use openssl rand -base64 33
MORTY_KEY=ReplaceWithARealKey!

2
.gitignore vendored
View file

@ -3,4 +3,4 @@
searxng-docker.service searxng-docker.service
caddy caddy
srv srv
searx searxng/uwsgi.ini

35
Caddyfile
View file

@ -11,7 +11,9 @@
@api { @api {
path /config path /config
path /status path /healthz
path /stats/errors
path /stats/checker
} }
@static { @static {
@ -22,12 +24,12 @@
not path /static/* not path /static/*
} }
@morty { @imageproxy {
path /morty/* path /image_proxy
} }
@notmorty { @notimageproxy {
not path /morty/* not path /image_proxy
} }
header { header {
@ -40,11 +42,8 @@
# Prevent some browsers from MIME-sniffing a response away from the declared Content-Type # Prevent some browsers from MIME-sniffing a response away from the declared Content-Type
X-Content-Type-Options "nosniff" X-Content-Type-Options "nosniff"
# Disallow the site to be rendered within a frame (clickjacking protection)
X-Frame-Options "SAMEORIGIN"
# Disable some features # Disable some features
Permissions-Policy "accelerometer=();ambient-light-sensor=(); autoplay=();camera=();encrypted-media=();focus-without-user-activation=(); geolocation=();gyroscope=();magnetometer=();microphone=();midi=();payment=();picture-in-picture=(); speaker=();sync-xhr=();usb=();vr=()" Permissions-Policy "accelerometer=(),ambient-light-sensor=(),autoplay=(),camera=(),encrypted-media=(),focus-without-user-activation=(),geolocation=(),gyroscope=(),magnetometer=(),microphone=(),midi=(),payment=(),picture-in-picture=(),speaker=(),sync-xhr=(),usb=(),vr=()"
# Disable some features (legacy) # Disable some features (legacy)
Feature-Policy "accelerometer 'none';ambient-light-sensor 'none'; autoplay 'none';camera 'none';encrypted-media 'none';focus-without-user-activation 'none'; geolocation 'none';gyroscope 'none';magnetometer 'none';microphone 'none';midi 'none';payment 'none';picture-in-picture 'none'; speaker 'none';sync-xhr 'none';usb 'none';vr 'none'" Feature-Policy "accelerometer 'none';ambient-light-sensor 'none'; autoplay 'none';camera 'none';encrypted-media 'none';focus-without-user-activation 'none'; geolocation 'none';gyroscope 'none';magnetometer 'none';microphone 'none';midi 'none';payment 'none';picture-in-picture 'none'; speaker 'none';sync-xhr 'none';usb 'none';vr 'none'"
@ -78,29 +77,21 @@
} }
# CSP (see http://content-security-policy.com/ ) # CSP (see http://content-security-policy.com/ )
header @morty { header @imageproxy {
Content-Security-Policy "default-src 'none'; style-src 'self' 'unsafe-inline'; form-action 'self'; frame-ancestors 'self'; base-uri 'self'; img-src 'self' data:; font-src 'self'; frame-src 'self'" Content-Security-Policy "default-src 'none'; img-src 'self' data:"
} }
header @notmorty { header @notimageproxy {
Content-Security-Policy "upgrade-insecure-requests; default-src 'none'; script-src 'self'; style-src 'self' 'unsafe-inline'; form-action 'self'; font-src 'self'; frame-ancestors 'self'; base-uri 'self'; connect-src 'self' https://overpass-api.de; img-src 'self' data: https://*.tile.openstreetmap.org; frame-src https://www.youtube-nocookie.com https://player.vimeo.com https://www.dailymotion.com https://www.deezer.com https://www.mixcloud.com https://w.soundcloud.com https://embed.spotify.com" Content-Security-Policy "upgrade-insecure-requests; default-src 'none'; script-src 'self'; style-src 'self' 'unsafe-inline'; form-action 'self'; font-src 'self'; frame-ancestors 'self'; base-uri 'self'; connect-src 'self' https://overpass-api.de; img-src 'self' data: https://*.tile.openstreetmap.org; frame-src https://www.youtube-nocookie.com https://player.vimeo.com https://www.dailymotion.com https://www.deezer.com https://www.mixcloud.com https://w.soundcloud.com https://embed.spotify.com"
} }
# Morty # SearXNG
handle @morty {
reverse_proxy localhost:3000
}
# Filtron
handle { handle {
encode zstd gzip encode zstd gzip
reverse_proxy localhost:4040 { reverse_proxy localhost:8080 {
header_up X-Forwarded-Port {http.request.port} header_up X-Forwarded-Port {http.request.port}
header_up X-Forwarded-Proto {http.request.scheme} header_up X-Forwarded-Proto {http.request.scheme}
header_up X-Forwarded-TlsProto {tls_protocol}
header_up X-Forwarded-TlsCipher {tls_cipher}
header_up X-Forwarded-HttpsProto {proto}
} }
} }

30
README.md
View file

@ -7,22 +7,22 @@ Create a new SearXNG instance in five minutes using Docker
| Name | Description | Docker image | Dockerfile | | Name | Description | Docker image | Dockerfile |
| -- | -- | -- | -- | | -- | -- | -- | -- |
| [Caddy](https://github.com/caddyserver/caddy) | Reverse proxy (create a LetsEncrypt certificate automatically) | [caddy/caddy:2-alpine](https://hub.docker.com/_/caddy) | [Dockerfile](https://github.com/caddyserver/caddy-docker) | | [Caddy](https://github.com/caddyserver/caddy) | Reverse proxy (create a LetsEncrypt certificate automatically) | [caddy/caddy:2-alpine](https://hub.docker.com/_/caddy) | [Dockerfile](https://github.com/caddyserver/caddy-docker) |
| [Filtron](https://github.com/dalf/filtron) | Filtering reverse HTTP proxy, bot and abuse protection | [dalf/filtron:latest](https://hub.docker.com/r/dalf/filtron) | See [asciimoo/filtron#4](https://github.com/asciimoo/filtron/pull/4) |
| [SearXNG](https://github.com/searxng/searxng) | SearXNG by itself | [searxng/searxng:latest](https://hub.docker.com/r/searxng/searxng) | [Dockerfile](https://github.com/searxng/searxng/blob/master/Dockerfile) | | [SearXNG](https://github.com/searxng/searxng) | SearXNG by itself | [searxng/searxng:latest](https://hub.docker.com/r/searxng/searxng) | [Dockerfile](https://github.com/searxng/searxng/blob/master/Dockerfile) |
| [Morty](https://github.com/dalf/morty) | Privacy aware web content sanitizer proxy as a service. | [dalf/morty:latest](https://hub.docker.com/r/dalf/morty) | [Dockerfile](https://github.com/dalf/morty/blob/master/Dockerfile) | | [Redis](https://github.com/redis/redis) | In-memory database | [redis:alpine](https://hub.docker.com/_/redis) | [Dockerfile-alpine.template](https://github.com/docker-library/redis/blob/master/Dockerfile-alpine.template) |
## How to use it ## How to use it
- [Install docker](https://docs.docker.com/install/) - [Install docker](https://docs.docker.com/install/)
- [Install docker-compose](https://docs.docker.com/compose/install/) (be sure that docker-compose version is at least 1.9.0) - [Install docker-compose](https://docs.docker.com/compose/install/) (be sure that docker-compose version is at least 1.9.0)
- only on MacOSX: ```brew install coreutils``` to install ```greadlink``` - only on MacOSX: ```brew install coreutils``` to install ```greadlink```
- Get searxng-docker: - Get searxng-docker
```sh ```sh
cd /usr/local cd /usr/local
git clone https://github.com/searxng/searxng-docker.git git clone https://github.com/searxng/searxng-docker.git
cd searxng-docker cd searxng-docker
``` ```
- Generate MORTY_KEY ```sed -i "s|ReplaceWithARealKey\!|$(openssl rand -base64 33)|g" .env``` - Edit the [.env](https://github.com/searxng/searxng-docker/blob/master/.env) file to set the hostname and an email
- Edit the other settings in [.env](https://github.com/searxng/searxng-docker/blob/master/.env) file according to your need - Generate the secret key ```sed -i "s|ultrasecretkey|$(openssl rand -hex 32)|g" searxng/settings.yml```
- Edit the [searxng/settings.yml](https://github.com/searxng/searxng-docker/blob/master/searxng/settings.yml) file according to your need
- Check everything is working: ```./start.sh``` - Check everything is working: ```./start.sh```
- ```cp searxng-docker.service.template searxng-docker.service``` - ```cp searxng-docker.service.template searxng-docker.service```
- edit the content of ```WorkingDirectory``` in the ```searxng-docker.service``` file (only if the installation path is different from /usr/local/searxng-docker) - edit the content of ```WorkingDirectory``` in the ```searxng-docker.service``` file (only if the installation path is different from /usr/local/searxng-docker)
@ -34,9 +34,9 @@ Create a new SearXNG instance in five minutes using Docker
## Note on the image proxy feature ## Note on the image proxy feature
The SearXNG image proxy is activated by default using [Morty](https://github.com/dalf/morty). The SearXNG image proxy is activated by default.
The default [Content-Security-Policy](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Security-Policy) allow the browser to access to {SEARXNG_HOSTNAME} and ```https://*.tile.openstreetmap.org;```. The default [Content-Security-Policy](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Security-Policy) allow the browser to access to ```${SEARXNG_HOSTNAME}``` and ```https://*.tile.openstreetmap.org;```.
If some users wants to disable the image proxy, you have to modify [./Caddyfile](https://github.com/searxng/searxng-docker/blob/master/Caddyfile). Replace the ```img-src 'self' data: https://*.tile.openstreetmap.org;``` by ```img-src * data:;```. If some users wants to disable the image proxy, you have to modify [./Caddyfile](https://github.com/searxng/searxng-docker/blob/master/Caddyfile). Replace the ```img-src 'self' data: https://*.tile.openstreetmap.org;``` by ```img-src * data:;```.
@ -52,17 +52,11 @@ It is possible to use the [extend feature](https://docs.docker.com/compose/exten
## Multi Architecture Docker images ## Multi Architecture Docker images
For now only the amd64 platform is supported. Supported architecture:
- amd64
- arm64
- arm/v7
## How to update ? ## How to update ?
Check the content of [```update.sh```](https://github.com/searxng/searxng-docker/blob/master/update.sh). Check the content of [```update.sh```](https://github.com/searxng/searxng-docker/blob/master/update.sh)
## Access to the Filtron API
The [Filtron API](https://github.com/dalf/filtron#api) is available on ```http://localhost:4041```.
For example, to display the loaded rules:
```
curl http://localhost:4041/rules | jq
```

54
docker-compose.yaml
View file

@ -5,14 +5,13 @@ services:
caddy: caddy:
container_name: caddy container_name: caddy
image: caddy:2-alpine image: caddy:2-alpine
restart: on-failure
network_mode: host network_mode: host
volumes: volumes:
- ./Caddyfile:/etc/caddy/Caddyfile:ro - ./Caddyfile:/etc/caddy/Caddyfile:ro
- caddy-data:/data:rw - caddy-data:/data:rw
- caddy-config:/config:rw - caddy-config:/config:rw
environment: environment:
- SEARXNG_HOSTNAME=${SEARXNG_HOSTNAME:-localhost} - SEARXNG_HOSTNAME=${SEARXNG_HOSTNAME:-http://localhost:80}
- SEARXNG_TLS=${LETSENCRYPT_EMAIL:-internal} - SEARXNG_TLS=${LETSENCRYPT_EMAIL:-internal}
cap_drop: cap_drop:
- ALL - ALL
@ -20,21 +19,20 @@ services:
- NET_BIND_SERVICE - NET_BIND_SERVICE
- DAC_OVERRIDE - DAC_OVERRIDE
filtron: redis:
container_name: filtron container_name: redis
image: dalf/filtron image: "redis:alpine"
restart: always command: redis-server --save "" --appendonly "no"
ports:
- "127.0.0.1:4040:4040"
- "127.0.0.1:4041:4041"
networks: networks:
- searxng - searxng
command: -listen 0.0.0.0:4040 -api 0.0.0.0:4041 -target searxng:8080 tmpfs:
volumes: - /var/lib/redis
- ./rules.json:/etc/filtron/rules.json:rw
read_only: true
cap_drop: cap_drop:
- ALL - ALL
cap_add:
- SETGID
- SETUID
- DAC_OVERRIDE
searxng: searxng:
container_name: searxng container_name: searxng
@ -42,14 +40,12 @@ services:
restart: always restart: always
networks: networks:
- searxng - searxng
command: ${SEARXNG_COMMAND:-} ports:
- "127.0.0.1:8080:8080"
volumes: volumes:
- ./searxng:/etc/searxng:rw - ./searxng:/etc/searxng:rw
environment: environment:
- BIND_ADDRESS=0.0.0.0:8080 - SEARXNG_BASE_URL=https://${SEARXNG_HOSTNAME:-localhost}/
- BASE_URL=https://${SEARXNG_HOSTNAME:-localhost}/
- MORTY_URL=https://${SEARXNG_HOSTNAME:-localhost}/morty/
- MORTY_KEY=${MORTY_KEY}
cap_drop: cap_drop:
- ALL - ALL
cap_add: cap_add:
@ -57,25 +53,11 @@ services:
- SETGID - SETGID
- SETUID - SETUID
- DAC_OVERRIDE - DAC_OVERRIDE
morty:
container_name: morty
image: dalf/morty
restart: always
ports:
- "127.0.0.1:3000:3000"
networks:
- searxng
command: -timeout 6 -ipv6
environment:
- MORTY_KEY=${MORTY_KEY}
- MORTY_ADDRESS=0.0.0.0:3000
logging: logging:
driver: none driver: "json-file"
read_only: true options:
cap_drop: max-size: "1m"
- ALL max-file: "1"
networks: networks:
searxng: searxng:
ipam: ipam:

147
rules.json
View file

@ -1,147 +0,0 @@
[
{
"name": "searx.space",
"filters": ["Header:X-Forwarded-For=nslookup(check.searx.space)"],
"stop": true,
"actions": [{ "name": "log"}]
},
{
"name": "IP limit, all paths",
"interval": 3,
"limit": 25,
"aggregations": ["Header:X-Forwarded-For"],
"actions": [
{"name": "block",
"params": {"message": "Rate limit exceeded, try again later."}}
]
},
{
"name": "useragent limit, all paths",
"interval": 30,
"limit": 200,
"aggregations": ["Header:X-Forwarded-For", "Header:User-Agent"],
"stop": true,
"actions": [
{"name": "block",
"params": {"message": "Rate limit exceeded, try again later."}}
]
},
{
"name": "search request",
"filters": ["Param:q", "Path=^(/|/search)$"],
"subrules": [
{
"name": "allow Firefox Android (issue #48 and #60)",
"filters": [
"Param:q=^1$",
"Header:User-Agent=(^MozacFetch/[0-9]{2,3}.[0-9].[0-9]+$|^Mozilla/5.0 \\(Android [0-9]{1,2}(.[0-9]{1,2}.[0-9]{1,2})?; Mobile; rv:[0-9]{2,3}.[0-9]\\) Gecko/[0-9]{2,3}.[0-9] Firefox/[0-9]{2,3}.[0-9]$)"
],
"stop": true,
"actions": [{"name": "log"}]
},
{
"name": "robot agent forbidden",
"limit": 0,
"stop": true,
"filters": ["Header:User-Agent=([Cc][Uu][Rr][Ll]|[wW]get|Scrapy|splash|JavaFX|FeedFetcher|python-requests|Go-http-client|Java|Jakarta|okhttp|HttpClient|Jersey|Python|libwww-perl|Ruby|SynHttpClient|UniversalFeedParser)"],
"actions": [
{"name": "block",
"params": {"message": "Rate limit exceeded"}}
]
},
{
"name": "bot forbidden",
"limit": 0,
"stop": true,
"filters": ["Header:User-Agent=(Googlebot|GoogleImageProxy|bingbot|Baiduspider|yacybot|YandexMobileBot|YandexBot|Yahoo! Slurp|MJ12bot|AhrefsBot|archive.org_bot|msnbot|MJ12bot|SeznamBot|linkdexbot|Netvibes|SMTBot|zgrab|James BOT|Sogou|Abonti|Pixray|Spinn3r|SemrushBot|Exabot|ZmEu|BLEXBot|bitlybot)"],
"actions": [
{"name": "block",
"params": {"message": "Rate limit exceeded"}}
]
},
{
"name": "block missing accept-language",
"filters": ["!Header:Accept-Language"],
"limit": 0,
"stop": true,
"actions": [
{"name": "block",
"params": {"message": "Rate limit exceeded"}}
]
},
{
"name": "block Connection:close",
"filters": ["Header:Connection=close"],
"limit": 0,
"stop": true,
"actions": [
{"name": "block",
"params": {"message": "Rate limit exceeded"}}
]
},
{
"name": "block no gzip support",
"filters": ["!Header:Accept-Encoding=(^gzip$|^gzip[;,]|[; ]gzip$|[; ]gzip[;,])"],
"limit": 0,
"stop": true,
"actions": [
{"name": "block",
"params": {"message": "Rate limit exceeded"}}
]
},
{
"name": "block no deflate support",
"filters": ["!Header:Accept-Encoding=(^deflate$|^deflate[;,]|[; ]deflate$|[; ]deflate[;,])"],
"limit": 0,
"stop": true,
"actions": [
{"name": "block",
"params": {"message": "Rate limit exceeded"}}
]
},
{
"name": "block accept everything",
"filters": ["!Header:Accept=text/html"],
"limit": 0,
"stop": true,
"actions": [
{"name": "block",
"params": {"message": "Rate limit exceeded"}}
]
},
{
"name": "rss/json limit",
"interval": 3600,
"limit": 4,
"stop": true,
"filters": ["Param:format=(csv|json|rss)"],
"aggregations": ["Header:X-Forwarded-For"],
"actions": [
{"name": "block",
"params": {"message": "Rate limit exceeded, try again later."}}
]
},
{
"name": "IP limit",
"interval": 3,
"limit": 3,
"aggregations": ["Header:X-Forwarded-For"],
"actions": [
{"name": "block",
"params": {"message": "Rate limit exceeded, try again later."}}
]
},
{
"name": "IP and useragent limit",
"interval": 600,
"limit": 60,
"stop": true,
"aggregations": ["Header:X-Forwarded-For", "Header:User-Agent"],
"actions": [
{"name": "block",
"params": {"message": "Rate limit exceeded, try again later."}}
]
}
]
}
]

11
searxng/settings.yml Normal file
View file

@ -0,0 +1,11 @@
# see https://docs.searxng.org/admin/engines/settings.html#use-default-settings
use_default_settings: true
server:
# base_url is defined in the SEARXNG_BASE_URL environment variable, see .env and docker-compose.yml
secret_key: "ultrasecretkey" # change this!
limiter: true # can be disabled for a private instance
image_proxy: true
ui:
static_use_hash: true
redis:
url: redis://redis:6379/0