OwlBurst/Ponysearch
1
0
Fork 0
forked from Ponysearch/Ponysearch
Code Pull requests Activity

utils/filtron.sh: various fix from first installation test (WIP)

Browse source 
Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
This commit is contained in:
Markus Heiser 2020-01-09 16:25:05 +01:00
parent f20193155a
commit 4990b07b4b
4 changed files with 206 additions and 85 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

75
utils/filtron.sh
View file

@ -11,6 +11,11 @@ source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
FILTRON_ETC="/etc/filtron" FILTRON_ETC="/etc/filtron"
FILTRON_RULES="$FILTRON_ETC/rules.json"
FILTRON_API="127.0.0.1:4005"
FILTRON_LISTEN="127.0.0.1:4004"
FILTRON_TARGET="127.0.0.1:8888"
SERVICE_NAME="filtron" SERVICE_NAME="filtron"
SERVICE_USER="${SERVICE_NAME}" SERVICE_USER="${SERVICE_NAME}"
SERVICE_HOME="/home/${SERVICE_USER}" SERVICE_HOME="/home/${SERVICE_USER}"
@ -23,6 +28,11 @@ GO_ENV="${SERVICE_HOME}/.go_env"
GO_PKG_URL="https://dl.google.com/go/go1.13.5.linux-amd64.tar.gz" GO_PKG_URL="https://dl.google.com/go/go1.13.5.linux-amd64.tar.gz"
GO_TAR=$(basename "$GO_PKG_URL") GO_TAR=$(basename "$GO_PKG_URL")
CONFIG_FILES=(
"${FILTRON_RULES}"
"${SERVICE_SYSTEMD_UNIT}"
)
# ---------------------------------------------------------------------------- # ----------------------------------------------------------------------------
usage(){ usage(){
# ---------------------------------------------------------------------------- # ----------------------------------------------------------------------------
@ -37,10 +47,16 @@ usage:
$(basename "$0") remove [all] $(basename "$0") remove [all]
$(basename "$0") activate [server] $(basename "$0") activate [server]
$(basename "$0") deactivate [server] $(basename "$0") deactivate [server]
$(basename "$0") show [server]
shell - start interactive shell with user ${SERVICE_USER} shell
install user - add service user '$SERVICE_USER' at $SERVICE_HOME start interactive shell from user ${SERVICE_USER}
show server
show server status and log
install / remove
all - complete setup of filtron server
install user
add service user '$SERVICE_USER' at $SERVICE_HOME
EOF EOF
[ ! -z ${1+x} ] && echo -e "$1" [ ! -z ${1+x} ] && echo -e "$1"
} }
@ -58,6 +74,14 @@ main(){
sudo_or_exit sudo_or_exit
interactive_shell interactive_shell
;; ;;
show)
case $2 in
server)
sudo_or_exit
show_server
;;
*) usage "$_usage"; exit 42;;
esac ;;
install) install)
sudo_or_exit sudo_or_exit
case $2 in case $2 in
@ -91,21 +115,27 @@ main(){
install_all() { install_all() {
rst_title "Install $SERVICE_NAME (service)" rst_title "Install $SERVICE_NAME (service)"
assert_user assert_user
wait_key
install_go install_go
wait_key
install_filtron install_filtron
wait_key
install_server install_server
wait_key
} }
remove_all() { remove_all() {
rst_title "De-Install $SERVICE_NAME (service)" rst_title "De-Install $SERVICE_NAME (service)"
remove_server remove_server
wait_key
remove_user remove_user
rm -rf "$FILTRON_ETC" rm -r "$FILTRON_ETC" 2>&1 | prefix_stdout
wait_key wait_key
} }
install_server() { install_server() {
rst_title "Install System-D Unit ${SERVICE_NAME}.service ..." section rst_title "Install System-D Unit ${SERVICE_NAME}.service" section
echo
install_template ${SERVICE_SYSTEMD_UNIT} root root 644 install_template ${SERVICE_SYSTEMD_UNIT} root root 644
wait_key wait_key
activate_server activate_server
@ -116,12 +146,12 @@ remove_server() {
return return
fi fi
deactivate_server deactivate_server
rm "${SERVICE_SYSTEMD_UNIT}" rm "${SERVICE_SYSTEMD_UNIT}" 2>&1 | prefix_stdout
} }
activate_server () { activate_server () {
rst_title "Activate $SERVICE_NAME (service)" section rst_title "Activate $SERVICE_NAME (service)" section
echo
tee_stderr <<EOF | bash 2>&1 | prefix_stdout tee_stderr <<EOF | bash 2>&1 | prefix_stdout
systemctl enable $SERVICE_NAME.service systemctl enable $SERVICE_NAME.service
systemctl restart $SERVICE_NAME.service systemctl restart $SERVICE_NAME.service
@ -129,7 +159,6 @@ EOF
tee_stderr <<EOF | bash 2>&1 | prefix_stdout tee_stderr <<EOF | bash 2>&1 | prefix_stdout
systemctl status $SERVICE_NAME.service systemctl status $SERVICE_NAME.service
EOF EOF
wait_key
} }
deactivate_server () { deactivate_server () {
@ -139,7 +168,6 @@ deactivate_server () {
systemctl stop $SERVICE_NAME.service systemctl stop $SERVICE_NAME.service
systemctl disable $SERVICE_NAME.service systemctl disable $SERVICE_NAME.service
EOF EOF
wait_key
} }
assert_user() { assert_user() {
@ -168,18 +196,18 @@ EOF
remove_user() { remove_user() {
rst_title "Drop $SERVICE_USER HOME" section rst_title "Drop $SERVICE_USER HOME" section
if ask_yn "Do you really want to drop $SERVICE_USER home folder?"; then if ask_yn "Do you really want to drop $SERVICE_USER home folder?"; then
userdel -r -f "$SERVICE_USER" userdel -r -f "$SERVICE_USER" 2>&1 | prefix_stdout
else else
rst_para "Leave HOME folder $(du -sh "$SERVICE_HOME") unchanged." rst_para "Leave HOME folder $(du -sh "$SERVICE_HOME") unchanged."
fi fi
} }
interactive_shell(){ interactive_shell(){
echo "// exit with STRG-D" echo "// exit with CTRL-D"
sudo -H -u ${SERVICE_USER} -i sudo -H -u ${SERVICE_USER} -i
} }
_service_prefix="$SERVICE_USER@$(hostname) -->| " _service_prefix=" |$SERVICE_USER| "
install_go(){ install_go(){
rst_title "Install Go in user's HOME" section rst_title "Install Go in user's HOME" section
@ -199,14 +227,29 @@ EOF
! which go >/dev/null && echo "Go Installation not found in PATH!?!" ! which go >/dev/null && echo "Go Installation not found in PATH!?!"
which go >/dev/null && go version && echo "congratulations -- Go installation OK :)" which go >/dev/null && go version && echo "congratulations -- Go installation OK :)"
EOF EOF
wait_key
} }
install_filtron() { install_filtron() {
tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" | prefix_stdout "$_service_prefix" rst_title "Install filtron in user's ~/go-apps" section
go get -v -u github.com/asciimoo/filtron 2>&1 echo
tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" 2>&1 | prefix_stdout "$_service_prefix"
go get -v -u github.com/asciimoo/filtron
EOF EOF
install_template "$FILTRON_ETC/rules.json" root root 644 install_template --no-eval "$FILTRON_RULES" root root 644
}
show_server () {
rst_title "server status & log"
echo
systemctl status filtron.service
echo
read -s -n1 -t 5 -p "// use CTRL-C to stop monitoring the log"
echo
while true; do
trap break 2
journalctl -f -u filtron
done
return 0
} }
# ---------------------------------------------------------------------------- # ----------------------------------------------------------------------------

72
utils/lib.sh
View file

@ -22,6 +22,13 @@ if [[ -z "$SYSTEMD_UNITS" ]]; then
SYSTEMD_UNITS="/lib/systemd/system" SYSTEMD_UNITS="/lib/systemd/system"
fi fi
if [[ -z ${DIFF_CMD} ]]; then
DIFF_CMD="diff -u"
if command -v colordiff >/dev/null; then
DIFF_CMD="colordiff -u"
fi
fi
sudo_or_exit() { sudo_or_exit() {
# usage: sudo_or_exit # usage: sudo_or_exit
@ -55,10 +62,10 @@ rst_para() {
# usage: RST_INDENT=1 rst_para "lorem ipsum ..." # usage: RST_INDENT=1 rst_para "lorem ipsum ..."
local prefix='' local prefix=''
if ! [[ -z $RST_INDENT ]] && [[ $RST_INDENT -gt 0 ]]; then if ! [[ -z $RST_INDENT ]] && [[ $RST_INDENT -gt 0 ]]; then
prefix="$(for i in $(seq 1 "$RST_INDENT"); do printf " "; done)" prefix="$(for i in $(seq 1 "$RST_INDENT"); do printf " "; done)"
echo -en "\n$*\n" | $FMT | prefix_stdout "$prefix" echo -en "\n$*\n" | $FMT | prefix_stdout "$prefix"
else else
echo -en "\n$*\n" | $FMT echo -en "\n$*\n" | $FMT
fi fi
} }
@ -66,15 +73,23 @@ err_msg() { echo -e "ERROR: $*" >&2; }
warn_msg() { echo -e "WARN: $*" >&2; } warn_msg() { echo -e "WARN: $*" >&2; }
info_msg() { echo -e "INFO: $*"; } info_msg() { echo -e "INFO: $*"; }
clean_stdin() {
if [[ $(uname -s) != 'Darwin' ]]; then
while $(read -n1 -t 0.1); do : ; done
fi
}
wait_key(){ wait_key(){
# usage: waitKEY [<timeout in sec>] # usage: waitKEY [<timeout in sec>]
clean_stdin
local _t=$1 local _t=$1
[[ ! -z $FORCE_TIMEOUT ]] && _t=$FORCE_TIMEOUT [[ ! -z $FORCE_TIMEOUT ]] && _t=$FORCE_TIMEOUT
[[ ! -z $_t ]] && _t="-t $_t" [[ ! -z $_t ]] && _t="-t $_t"
# shellcheck disable=SC2086 # shellcheck disable=SC2086
read -n1 $_t -p "** press any [KEY] to continue **" read -s -n1 $_t -p "** press any [KEY] to continue **"
echo echo
clean_stdin
} }
ask_yn() { ask_yn() {
@ -100,6 +115,7 @@ ask_yn() {
esac esac
echo echo
while true; do while true; do
clean_stdin
printf "$1 ${choice} " printf "$1 ${choice} "
# shellcheck disable=SC2086 # shellcheck disable=SC2086
read -n1 $_t read -n1 $_t
@ -117,6 +133,7 @@ ask_yn() {
_t="" _t=""
err_msg "invalid choice" err_msg "invalid choice"
done done
clean_stdin
return $exit_val return $exit_val
} }
@ -144,7 +161,7 @@ tee_stderr () {
prefix_stdout () { prefix_stdout () {
# usage: <cmd> | prefix_stdout [prefix] # usage: <cmd> | prefix_stdout [prefix]
local prefix="-->| " local prefix=" | "
if [[ ! -z $1 ]] ; then prefix="$1"; fi if [[ ! -z $1 ]] ; then prefix="$1"; fi
@ -223,6 +240,7 @@ choose_one() {
fi fi
done done
while true; do while true; do
clean_stdin
printf "$1 [$default] " printf "$1 [$default] "
if (( 10 > $max )); then if (( 10 > $max )); then
@ -242,6 +260,7 @@ choose_one() {
err_msg "invalid choice" err_msg "invalid choice"
done done
echo echo
clean_stdin
eval "$env_name"='${list[${REPLY}]}' eval "$env_name"='${list[${REPLY}]}'
} }
@ -288,31 +307,48 @@ install_template() {
mkdir -p "$(dirname "${dst}")" mkdir -p "$(dirname "${dst}")"
if [[ -f "${dst}" ]] ; then if [[ ! -f "${dst}" ]]; then
info_msg "file ${dst} allready exists on this host" info_msg "install: ${template_file}"
sudo -H install -v -o "${owner}" -g "${group}" -m "${chmod}" \
"${template_file}" "${dst}" | prefix_stdout
return $?
fi
if [[ -f "${dst}" ]] && cmp --silent "${template_file}" "${dst}" ; then
info_msg "file ${dst} allready installed"
return 0
fi
info_msg "file ${dst} allready exists on this host"
while true; do
choose_one _reply "choose next step with file $dst" \ choose_one _reply "choose next step with file $dst" \
"replace file" \ "replace file" \
"leave file unchanged" "leave file unchanged" \
"interactiv shell" \
"diff files"
case $_reply in case $_reply in
"replace file") "replace file")
info_msg "install: ${template_file}" info_msg "install: ${template_file}"
sudo -H install -v -o "${owner}" -g "${group}" -m "${chmod}" \ sudo -H install -v -o "${owner}" -g "${group}" -m "${chmod}" \
"${template_file}" "${dst}" | prefix_stdout "${template_file}" "${dst}" | prefix_stdout
break
;; ;;
"leave file unchanged") "leave file unchanged")
break
;; ;;
"interactiv shell") "interactiv shell")
echo "// exit with STRG-D" echo "// edit ${dst} to your needs"
echo "// exit with CTRL-D"
sudo -H -u "${owner}" -i sudo -H -u "${owner}" -i
;; $DIFF_CMD "${dst}" "${template_file}"
if ask_yn "did you edit ${template_file} to your needs?"; then
break
fi
;;
"diff files")
$DIFF_CMD "${dst}" "${template_file}" | prefix_stdout
esac esac
done
else
info_msg "install: ${template_file}"
sudo -H install -v -o "${owner}" -g "${group}" -m "${chmod}" \
"${template_file}" "${dst}" | prefix_stdout
fi
} }

142
utils/templates/etc/filtron/rules.json
View file

@ -1,56 +1,98 @@
[ [{
"name":"search request",
"filters":[
"Param:q",
"Path=^(/|/search)$"
],
"interval":60,
"limit":15,
"subrules":[
{ {
"name": "api limit", "name":"roboagent limit",
"interval": 60, "interval":60,
"limit": 1000, "limit":15,
"filters": ["Path=^/api"], "filters":[
"aggregations": ["Path"], "Header:User-Agent=(curl|cURL|Wget|python-requests|Scrapy|FeedFetcher|Go-http-client)"
"actions": [ ],
{"name": "block"} "actions":[
], {"name": "log"},
"subrules": [ {
{ "name":"block",
"name": "drop put", "params":{
"interval": 60, "message":"Rate limit exceeded"
"limit": 100, }
"filters": ["Method=PUT"], }
"aggregations": ["Header:X-Forwarded-For"], ]
"actions": [
{"name": "shell",
"params": {"cmd": "iptables -A INPUT -s %v -j DROP", "args": ["Header:X-Forwarded-For"]}}
]
}
]
}, },
{ {
"name": "log'n'block rss", "name":"botlimit",
"interval": 300, "limit":0,
"limit": 2500, "stop":true,
"filters": ["Path=^/$", "GET:format=rss"], "filters":[
"actions": [ "Header:User-Agent=(Googlebot|bingbot|Baiduspider|yacybot|YandexMobileBot|YandexBot|Yahoo! Slurp|MJ12bot|AhrefsBot|archive.org_bot|msnbot|MJ12bot|SeznamBot|linkdexbot|Netvibes|SMTBot|zgrab|James BOT)"
{"name": "log"}, ],
{"name": "block"} "actions":[
] {"name": "log"},
{
"name":"block",
"params":{
"message":"Rate limit exceeded"
}
}
]
}, },
{ {
"name": "log rule", "name":"IP limit",
"filters": ["Path=/"], "interval":60,
"actions": [ {"name": "log"} ], "limit":15,
"subrules": [ "stop":true,
{ "aggregations":[
"name": "block missing accept-language", "Header:X-Forwarded-For"
"filters": ["!Header:Accept-Language"], ],
"actions": [ "actions":[
{"name": "block"} {"name": "log"},
] {
}, "name":"block",
{ "params":{
"name": "block curl", "message":"Rate limit exceeded"
"filters": ["Header:User-Agent=[Cc]url"], }
"actions": [ }
{"name": "block"} ]
] },
} {
] "name":"rss/json limit",
"interval":60,
"limit":15,
"stop":true,
"filters":[
"Param:format=(csv|json|rss)"
],
"actions":[
{"name": "log"},
{
"name":"block",
"params":{
"message":"Rate limit exceeded"
}
}
]
},
{
"name":"useragent limit",
"interval":60,
"limit":15,
"aggregations":[
"Header:User-Agent"
],
"actions":[
{"name": "log"},
{
"name":"block",
"params":{
"message":"Rate limit exceeded"
}
}
]
} }
] ]
}]

2
utils/templates/lib/systemd/system/filtron.service
View file

@ -10,7 +10,7 @@ Type=simple
User=${SERVICE_USER} User=${SERVICE_USER}
Group=${SERVICE_GROUP} Group=${SERVICE_GROUP}
WorkingDirectory=${SERVICE_HOME} WorkingDirectory=${SERVICE_HOME}
ExecStart=${SERVICE_HOME}/go-apps/bin/filtron -rules ${FILTRON_RULES} ExecStart=${SERVICE_HOME}/go-apps/bin/filtron -api '${FILTRON_API}' -listen '${FILTRON_LISTEN}' -rules '${FILTRON_RULES}' -target '${FILTRON_TARGET}'
Restart=always Restart=always
Environment=USER=${SERVICE_USER} HOME=${SERVICE_HOME} Environment=USER=${SERVICE_USER} HOME=${SERVICE_HOME}