From f18a3a77ef894d6563543fe449321e52b16e93ad Mon Sep 17 00:00:00 2001 From: Guilhem Bonnefille Date: Thu, 28 Apr 2016 13:59:41 +0200 Subject: [PATCH 1/3] Use new settings to fine tune the connection pool In some circumstances, it is necessary to increase size of pool. --- searx/poolrequests.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/searx/poolrequests.py b/searx/poolrequests.py index 13c6a906e..54e8dbcf6 100644 --- a/searx/poolrequests.py +++ b/searx/poolrequests.py @@ -41,14 +41,18 @@ class HTTPAdapterWithConnParams(requests.adapters.HTTPAdapter): block=self._pool_block, **self._conn_params) +connect=settings['outgoing'].get('pool_connections', 100) # Magic number kept from previous code +maxsize=settings['outgoing'].get('pool_maxsize', requests.adapters.DEFAULT_POOLSIZE) # Picked from constructor if settings['outgoing'].get('source_ips'): - http_adapters = cycle(HTTPAdapterWithConnParams(pool_connections=100, source_address=(source_ip, 0)) + http_adapters = cycle(HTTPAdapterWithConnParams(pool_connections=connect, pool_maxsize=maxsize, + source_address=(source_ip, 0)) for source_ip in settings['outgoing']['source_ips']) - https_adapters = cycle(HTTPAdapterWithConnParams(pool_connections=100, source_address=(source_ip, 0)) + https_adapters = cycle(HTTPAdapterWithConnParams(pool_connections=connect, pool_maxsize=maxsize, + source_address=(source_ip, 0)) for source_ip in settings['outgoing']['source_ips']) else: - http_adapters = cycle((HTTPAdapterWithConnParams(pool_connections=100), )) - https_adapters = cycle((HTTPAdapterWithConnParams(pool_connections=100), )) + http_adapters = cycle((HTTPAdapterWithConnParams(pool_connections=connect, pool_maxsize=maxsize), )) + https_adapters = cycle((HTTPAdapterWithConnParams(pool_connections=connect, pool_maxsize=maxsize), )) class SessionSinglePool(requests.Session): From 3f395304f96a7c71390dd0a79d0ba2b3c404d504 Mon Sep 17 00:00:00 2001 From: Guilhem Bonnefille Date: Mon, 2 May 2016 19:25:09 +0200 Subject: [PATCH 2/3] Fix pep8 --- searx/poolrequests.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/searx/poolrequests.py b/searx/poolrequests.py index 54e8dbcf6..f268df207 100644 --- a/searx/poolrequests.py +++ b/searx/poolrequests.py @@ -41,14 +41,14 @@ class HTTPAdapterWithConnParams(requests.adapters.HTTPAdapter): block=self._pool_block, **self._conn_params) -connect=settings['outgoing'].get('pool_connections', 100) # Magic number kept from previous code -maxsize=settings['outgoing'].get('pool_maxsize', requests.adapters.DEFAULT_POOLSIZE) # Picked from constructor +connect = settings['outgoing'].get('pool_connections', 100) # Magic number kept from previous code +maxsize = settings['outgoing'].get('pool_maxsize', requests.adapters.DEFAULT_POOLSIZE) # Picked from constructor if settings['outgoing'].get('source_ips'): http_adapters = cycle(HTTPAdapterWithConnParams(pool_connections=connect, pool_maxsize=maxsize, source_address=(source_ip, 0)) for source_ip in settings['outgoing']['source_ips']) https_adapters = cycle(HTTPAdapterWithConnParams(pool_connections=connect, pool_maxsize=maxsize, - source_address=(source_ip, 0)) + source_address=(source_ip, 0)) for source_ip in settings['outgoing']['source_ips']) else: http_adapters = cycle((HTTPAdapterWithConnParams(pool_connections=connect, pool_maxsize=maxsize), )) From d571a9d204dc4ff021bdc097dcaf87a5cc90dbe4 Mon Sep 17 00:00:00 2001 From: Guilhem Bonnefille Date: Mon, 2 May 2016 19:36:55 +0200 Subject: [PATCH 3/3] Preset new values with defaulting values from code --- searx/settings.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/searx/settings.yml b/searx/settings.yml index 462a0bcc2..c388f55ec 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -21,6 +21,8 @@ ui: outgoing: # communication with search engines request_timeout : 2.0 # seconds useragent_suffix : "" # suffix of searx_useragent, could contain informations like an email address to the administrator + pool_connections : 100 # Number of different hosts + pool_maxsize : 10 # Number of simultaneous requests by host # uncomment below section if you want to use a proxy # see http://docs.python-requests.org/en/latest/user/advanced/#proxies # SOCKS proxies are not supported : see https://github.com/kennethreitz/requests/pull/478