diff --git a/searx/plugins/limiter.py b/searx/plugins/limiter.py index 4402900e8..bdaeeea69 100644 --- a/searx/plugins/limiter.py +++ b/searx/plugins/limiter.py @@ -13,11 +13,11 @@ Enable the plugin in ``settings.yml``: - ``redis.url: ...`` check the value, see :ref:`settings redis` """ -import hmac import re from flask import request from searx.shared import redisdb +from searx.redislib import incr_sliding_window name = "Request limiter" description = "Limit the number of request" @@ -36,8 +36,9 @@ re_bot = re.compile( ) -def is_accepted_request(inc_get_counter) -> bool: +def is_accepted_request() -> bool: # pylint: disable=too-many-return-statements + redis_client = redisdb.client() user_agent = request.headers.get('User-Agent', '') x_forwarded_for = request.headers.get('X-Forwarded-For', '') @@ -47,70 +48,45 @@ def is_accepted_request(inc_get_counter) -> bool: return True if request.path == '/search': - c_burst = inc_get_counter(interval=20, keys=[b'IP limit, burst', x_forwarded_for]) - c_10min = inc_get_counter(interval=600, keys=[b'IP limit, 10 minutes', x_forwarded_for]) + c_burst = incr_sliding_window(redis_client, 'IP limit, burst' + x_forwarded_for, 20) + c_10min = incr_sliding_window(redis_client, 'IP limit, 10 minutes' + x_forwarded_for, 600) if c_burst > 15 or c_10min > 150: + logger.debug("to many request") # pylint: disable=undefined-variable return False if re_bot.match(user_agent): + logger.debug("detected bot") # pylint: disable=undefined-variable return False if len(request.headers.get('Accept-Language', '').strip()) == '': + logger.debug("missing Accept-Language") # pylint: disable=undefined-variable return False if request.headers.get('Connection') == 'close': + logger.debug("got Connection=close") # pylint: disable=undefined-variable return False accept_encoding_list = [l.strip() for l in request.headers.get('Accept-Encoding', '').split(',')] if 'gzip' not in accept_encoding_list or 'deflate' not in accept_encoding_list: + logger.debug("suspicious Accept-Encoding") # pylint: disable=undefined-variable return False if 'text/html' not in request.accept_mimetypes: + logger.debug("Accept-Encoding misses text/html") # pylint: disable=undefined-variable return False if request.args.get('format', 'html') != 'html': - c = inc_get_counter(interval=3600, keys=[b'API limit', x_forwarded_for]) + c = incr_sliding_window(redis_client, 'API limit' + x_forwarded_for, 3600) if c > 4: + logger.debug("API limit exceeded") # pylint: disable=undefined-variable return False return True -def create_inc_get_counter(redis_client, secret_key_bytes): - lua_script = """ - local slidingWindow = KEYS[1] - local key = KEYS[2] - local now = tonumber(redis.call('TIME')[1]) - local id = redis.call('INCR', 'counter') - if (id > 2^46) - then - redis.call('SET', 'count', 0) - end - redis.call('ZREMRANGEBYSCORE', key, 0, now - slidingWindow) - redis.call('ZADD', key, now, id) - local result = redis.call('ZCOUNT', key, 0, now+1) - redis.call('EXPIRE', key, slidingWindow) - return result - """ - script_sha = redis_client.script_load(lua_script) - - def inc_get_counter(interval, keys): - m = hmac.new(secret_key_bytes, digestmod='sha256') - for k in keys: - m.update(bytes(str(k), encoding='utf-8') or b'') - m.update(b"\0") - key = m.digest() - return redis_client.evalsha(script_sha, 2, interval, key) - - return inc_get_counter - - -def create_pre_request(get_aggregation_count): - def pre_request(): - if not is_accepted_request(get_aggregation_count): - return '', 429 - return None - - return pre_request +def pre_request(): + if not is_accepted_request(): + return '', 429 + return None def init(app, settings): @@ -122,8 +98,5 @@ def init(app, settings): logger.error("init limiter DB failed!!!") # pylint: disable=undefined-variable return False - redis_client = redisdb.client() - secret_key_bytes = bytes(settings['server']['secret_key'], encoding='utf-8') - inc_get_counter = create_inc_get_counter(redis_client, secret_key_bytes) - app.before_request(create_pre_request(inc_get_counter)) + app.before_request(pre_request) return True