OwlBurst/Ponysearch
1
0
Fork 0
forked from Ponysearch/Ponysearch
Code Pull requests Activity

/etc/filtron/rules.json: normalize rules from docs & tooling box

Browse source 
Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
This commit is contained in:
Markus Heiser 2020-02-04 17:59:58 +01:00
parent d07119ca49
commit 2f40f61f83
5 changed files with 210 additions and 207 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

32
docs/admin/filtron.rst
View file

@ -39,6 +39,10 @@ an installation and the maintenance of, use our script :ref:`filtron.sh`.
Sample configuration of filtron Sample configuration of filtron
=============================== ===============================
.. sidebar:: Tooling box
- :origin:`/etc/filtron/rules.json <utils/templates/etc/filtron/rules.json>`
An example configuration can be find below. This configuration limits the access An example configuration can be find below. This configuration limits the access
of: of:
@ -50,8 +54,8 @@ of:
.. code:: json .. code:: json
[{ [
"name":"search request", { "name": "search request",
"filters": [ "filters": [
"Param:q", "Param:q",
"Path=^(/|/search)$" "Path=^(/|/search)$"
@ -67,8 +71,8 @@ of:
"Header:User-Agent=(curl|cURL|Wget|python-requests|Scrapy|FeedFetcher|Go-http-client)" "Header:User-Agent=(curl|cURL|Wget|python-requests|Scrapy|FeedFetcher|Go-http-client)"
], ],
"actions": [ "actions": [
{ { "name": "log"},
"name":"block", { "name": "block",
"params": { "params": {
"message": "Rate limit exceeded" "message": "Rate limit exceeded"
} }
@ -83,8 +87,8 @@ of:
"Header:User-Agent=(Googlebot|bingbot|Baiduspider|yacybot|YandexMobileBot|YandexBot|Yahoo! Slurp|MJ12bot|AhrefsBot|archive.org_bot|msnbot|MJ12bot|SeznamBot|linkdexbot|Netvibes|SMTBot|zgrab|James BOT)" "Header:User-Agent=(Googlebot|bingbot|Baiduspider|yacybot|YandexMobileBot|YandexBot|Yahoo! Slurp|MJ12bot|AhrefsBot|archive.org_bot|msnbot|MJ12bot|SeznamBot|linkdexbot|Netvibes|SMTBot|zgrab|James BOT)"
], ],
"actions": [ "actions": [
{ { "name": "log"},
"name":"block", { "name": "block",
"params": { "params": {
"message": "Rate limit exceeded" "message": "Rate limit exceeded"
} }
@ -100,8 +104,8 @@ of:
"Header:X-Forwarded-For" "Header:X-Forwarded-For"
], ],
"actions": [ "actions": [
{ { "name": "log"},
"name":"block", { "name": "block",
"params": { "params": {
"message": "Rate limit exceeded" "message": "Rate limit exceeded"
} }
@ -117,8 +121,8 @@ of:
"Param:format=(csv|json|rss)" "Param:format=(csv|json|rss)"
], ],
"actions": [ "actions": [
{ { "name": "log"},
"name":"block", { "name": "block",
"params": { "params": {
"message": "Rate limit exceeded" "message": "Rate limit exceeded"
} }
@ -133,8 +137,8 @@ of:
"Header:User-Agent" "Header:User-Agent"
], ],
"actions": [ "actions": [
{ { "name": "log"},
"name":"block", { "name": "block",
"params": { "params": {
"message": "Rate limit exceeded" "message": "Rate limit exceeded"
} }
@ -142,8 +146,8 @@ of:
] ]
} }
] ]
}] }
]
Route request through filtron Route request through filtron

7
utils/filtron.sh
View file

@ -94,8 +94,9 @@ If needed, set PUBLIC_URL of your WEB service in the '${DOT_CONFIG#"$REPO_ROOT/"
FILTRON_API : ${FILTRON_API} FILTRON_API : ${FILTRON_API}
FILTRON_LISTEN : ${FILTRON_LISTEN} FILTRON_LISTEN : ${FILTRON_LISTEN}
FILTRON_TARGET : ${FILTRON_TARGET} FILTRON_TARGET : ${FILTRON_TARGET}
EOF EOF
[ ! -z ${1+x} ] && echo -e "$1" [ ! -z ${1+x} ] && err_msg "$1"
} }
main() { main() {
@ -105,7 +106,7 @@ main() {
dpkg apt-get install git wget curl \ dpkg apt-get install git wget curl \
|| exit || exit
local _usage="ERROR: unknown or missing $1 command $2" local _usage="unknown or missing $1 command $2"
case $1 in case $1 in
--source-only) ;; --source-only) ;;
@ -170,7 +171,7 @@ main() {
*) usage "$_usage"; exit 42;; *) usage "$_usage"; exit 42;;
esac ;; esac ;;
*) usage "ERROR: unknown or missing command $1"; exit 42;; *) usage "unknown or missing command $1"; exit 42;;
esac esac
} }

2
utils/morty.sh
View file

@ -84,7 +84,7 @@ ${DOT_CONFIG#"$REPO_ROOT/"} file::
SERVICE_USER : ${SERVICE_USER} SERVICE_USER : ${SERVICE_USER}
EOF EOF
info_searx info_searx
[ ! -z ${1+x} ] && echo -e "$1" [ ! -z ${1+x} ] && err_msg "$1"
} }
info_searx() { info_searx() {

6
utils/searx.sh
View file

@ -110,7 +110,7 @@ If needed, set PUBLIC_URL of your WEB service in the '${DOT_CONFIG#"$REPO_ROOT/"
SERVICE_USER : ${SERVICE_USER} SERVICE_USER : ${SERVICE_USER}
EOF EOF
[ ! -z ${1+x} ] && echo -e "$1" [ ! -z ${1+x} ] && err_msg "$1"
} }
main() { main() {
@ -120,7 +120,7 @@ main() {
dpkg systemctl apt-get install git wget curl \ dpkg systemctl apt-get install git wget curl \
|| exit || exit
local _usage="ERROR: unknown or missing $1 command $2" local _usage="unknown or missing $1 command $2"
case $1 in case $1 in
--source-only) ;; --source-only) ;;
@ -190,7 +190,7 @@ main() {
*) usage "$_usage"; exit 42;; *) usage "$_usage"; exit 42;;
esac ;; esac ;;
*) usage "ERROR: unknown or missing command $1"; exit 42;; *) usage "unknown or missing command $1"; exit 42;;
esac esac
} }

74
utils/templates/etc/filtron/rules.json
View file

@ -1,40 +1,41 @@
[{ [
"name":"search request", { "name": "suspiciously frequent IP",
"filters": [],
"interval": 600,
"limit": 30,
"aggregations": [
"Header:X-Forwarded-For"
],
"actions":[
{"name":"log"}
]
},
{ "name": "search request",
"filters": [ "filters": [
"Param:q", "Param:q",
"Path=^(/|/search)$" "Path=^(/|/search)$"
], ],
"interval":120, "interval": 61,
"limit":19, "limit": 999,
"actions":[
{
"name":"block",
"params":{
"message":"common rate limit exceeded"
}
}
],
"subrules": [ "subrules": [
{ {
"name": "roboagent limit", "name": "roboagent limit",
"interval":60, "interval": 61,
"limit":3, "limit": 1,
"filters": [ "filters": [
"Header:User-Agent=(curl|cURL|Wget|python-requests|Scrapy|FeedFetcher|Go-http-client|Ruby)" "Header:User-Agent=(curl|cURL|Wget|python-requests|Scrapy|FeedFetcher|Go-http-client)"
], ],
"actions": [ "actions": [
{ "name": "log"}, { "name": "log"},
{ { "name": "block",
"name":"block",
"params": { "params": {
"message":"rate limit exceeded" "message": "Rate limit exceeded"
} }
} }
] ]
}, },
{ {
"name": "botlimit", "name": "botlimit",
"interval":60,
"limit": 0, "limit": 0,
"stop": true, "stop": true,
"filters": [ "filters": [
@ -42,66 +43,63 @@
], ],
"actions": [ "actions": [
{ "name": "log"}, { "name": "log"},
{ { "name": "block",
"name":"block",
"params": { "params": {
"message":"rate limit exceeded" "message": "Rate limit exceeded"
} }
} }
] ]
}, },
{ {
"name": "IP limit", "name": "IP limit",
"interval":60, "interval": 61,
"limit":13, "limit": 9,
"stop": true, "stop": true,
"aggregations": [ "aggregations": [
"Header:X-Forwarded-For" "Header:X-Forwarded-For"
], ],
"actions": [ "actions": [
{ "name": "log"}, { "name": "log"},
{ { "name": "block",
"name":"block",
"params": { "params": {
"message":"rate limit exceeded" "message": "Rate limit exceeded"
} }
} }
] ]
}, },
{ {
"name": "rss/json limit", "name": "rss/json limit",
"interval":60, "interval": 121,
"limit":13, "limit": 2,
"stop": true, "stop": true,
"filters": [ "filters": [
"Param:format=(csv|json|rss)" "Param:format=(csv|json|rss)"
], ],
"actions": [ "actions": [
{ "name": "log"}, { "name": "log"},
{ { "name": "block",
"name":"block",
"params": { "params": {
"message":"rate limit exceeded" "message": "Rate limit exceeded"
} }
} }
] ]
}, },
{ {
"name": "useragent limit", "name": "useragent limit",
"interval":60, "interval": 61,
"limit":13, "limit": 199,
"aggregations": [ "aggregations": [
"Header:User-Agent" "Header:User-Agent"
], ],
"actions": [ "actions": [
{ "name": "log"}, { "name": "log"},
{ { "name": "block",
"name":"block",
"params": { "params": {
"message":"rate limit exceeded" "message": "Rate limit exceeded"
} }
} }
] ]
} }
] ]
}] }
]