forked from Ponysearch/Ponysearch
[pylint] searx/network/network.py & add global (NETWORKS)
No functional change! - fix messages from pylint - add ``global NETWORKS`` Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
This commit is contained in:
parent
b595c482d0
commit
44efa911ba
1 changed files with 26 additions and 14 deletions
|
@ -1,4 +1,7 @@
|
||||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
# lint: pylint
|
||||||
|
# pylint: disable=global-statement
|
||||||
|
# pylint: disable=missing-module-docstring, missing-class-docstring, missing-function-docstring
|
||||||
|
|
||||||
import atexit
|
import atexit
|
||||||
import asyncio
|
import asyncio
|
||||||
|
@ -39,7 +42,9 @@ class Network:
|
||||||
'local_addresses', 'proxies', 'max_redirects', 'retries', 'retry_on_http_error',
|
'local_addresses', 'proxies', 'max_redirects', 'retries', 'retry_on_http_error',
|
||||||
'_local_addresses_cycle', '_proxies_cycle', '_clients')
|
'_local_addresses_cycle', '_proxies_cycle', '_clients')
|
||||||
|
|
||||||
def __init__(self,
|
def __init__(
|
||||||
|
# pylint: disable=too-many-arguments
|
||||||
|
self,
|
||||||
enable_http=True,
|
enable_http=True,
|
||||||
verify=True,
|
verify=True,
|
||||||
enable_http2=False,
|
enable_http2=False,
|
||||||
|
@ -50,7 +55,8 @@ class Network:
|
||||||
local_addresses=None,
|
local_addresses=None,
|
||||||
retries=0,
|
retries=0,
|
||||||
retry_on_http_error=None,
|
retry_on_http_error=None,
|
||||||
max_redirects=30):
|
max_redirects=30 ):
|
||||||
|
|
||||||
self.enable_http = enable_http
|
self.enable_http = enable_http
|
||||||
self.verify = verify
|
self.verify = verify
|
||||||
self.enable_http2 = enable_http2
|
self.enable_http2 = enable_http2
|
||||||
|
@ -81,7 +87,7 @@ class Network:
|
||||||
local_addresses = self.local_addresses
|
local_addresses = self.local_addresses
|
||||||
if not local_addresses:
|
if not local_addresses:
|
||||||
return
|
return
|
||||||
elif isinstance(local_addresses, str):
|
if isinstance(local_addresses, str):
|
||||||
local_addresses = [local_addresses]
|
local_addresses = [local_addresses]
|
||||||
for address in local_addresses:
|
for address in local_addresses:
|
||||||
yield address
|
yield address
|
||||||
|
@ -119,6 +125,7 @@ class Network:
|
||||||
for pattern, proxy_urls in self.iter_proxies():
|
for pattern, proxy_urls in self.iter_proxies():
|
||||||
proxy_settings[pattern] = cycle(proxy_urls)
|
proxy_settings[pattern] = cycle(proxy_urls)
|
||||||
while True:
|
while True:
|
||||||
|
# pylint: disable=stop-iteration-return
|
||||||
yield tuple((pattern, next(proxy_url_cycle)) for pattern, proxy_url_cycle in proxy_settings.items())
|
yield tuple((pattern, next(proxy_url_cycle)) for pattern, proxy_url_cycle in proxy_settings.items())
|
||||||
|
|
||||||
def get_client(self, verify=None, max_redirects=None):
|
def get_client(self, verify=None, max_redirects=None):
|
||||||
|
@ -158,6 +165,7 @@ class Network:
|
||||||
return kwargs_clients
|
return kwargs_clients
|
||||||
|
|
||||||
def is_valid_respones(self, response):
|
def is_valid_respones(self, response):
|
||||||
|
# pylint: disable=too-many-boolean-expressions
|
||||||
if (self.retry_on_http_error is True and 400 <= response.status_code <= 599) \
|
if (self.retry_on_http_error is True and 400 <= response.status_code <= 599) \
|
||||||
or (isinstance(self.retry_on_http_error, list) and response.status_code in self.retry_on_http_error) \
|
or (isinstance(self.retry_on_http_error, list) and response.status_code in self.retry_on_http_error) \
|
||||||
or (isinstance(self.retry_on_http_error, int) and response.status_code == self.retry_on_http_error):
|
or (isinstance(self.retry_on_http_error, int) and response.status_code == self.retry_on_http_error):
|
||||||
|
@ -194,6 +202,7 @@ class Network:
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
async def aclose_all(cls):
|
async def aclose_all(cls):
|
||||||
|
global NETWORKS
|
||||||
await asyncio.gather(*[network.aclose() for network in NETWORKS.values()], return_exceptions=False)
|
await asyncio.gather(*[network.aclose() for network in NETWORKS.values()], return_exceptions=False)
|
||||||
|
|
||||||
|
|
||||||
|
@ -203,8 +212,10 @@ def get_network(name=None):
|
||||||
|
|
||||||
|
|
||||||
def initialize(settings_engines=None, settings_outgoing=None):
|
def initialize(settings_engines=None, settings_outgoing=None):
|
||||||
|
# pylint: disable=import-outside-toplevel)
|
||||||
from searx.engines import engines
|
from searx.engines import engines
|
||||||
from searx import settings
|
from searx import settings
|
||||||
|
# pylint: enable=import-outside-toplevel)
|
||||||
|
|
||||||
global NETWORKS
|
global NETWORKS
|
||||||
|
|
||||||
|
@ -212,7 +223,7 @@ def initialize(settings_engines=None, settings_outgoing=None):
|
||||||
settings_outgoing = settings_outgoing or settings.get('outgoing')
|
settings_outgoing = settings_outgoing or settings.get('outgoing')
|
||||||
|
|
||||||
# default parameters for AsyncHTTPTransport
|
# default parameters for AsyncHTTPTransport
|
||||||
# see https://github.com/encode/httpx/blob/e05a5372eb6172287458b37447c30f650047e1b8/httpx/_transports/default.py#L108-L121 # noqa
|
# see https://github.com/encode/httpx/blob/e05a5372eb6172287458b37447c30f650047e1b8/httpx/_transports/default.py#L108-L121 # pylint: disable=line-too-long
|
||||||
default_params = {
|
default_params = {
|
||||||
'enable_http': False,
|
'enable_http': False,
|
||||||
'verify': True,
|
'verify': True,
|
||||||
|
@ -290,6 +301,7 @@ def done():
|
||||||
Note: since Network.aclose has to be async, it is not possible to call this method on Network.__del__
|
Note: since Network.aclose has to be async, it is not possible to call this method on Network.__del__
|
||||||
So Network.aclose is called here using atexit.register
|
So Network.aclose is called here using atexit.register
|
||||||
"""
|
"""
|
||||||
|
global NETWORKS
|
||||||
try:
|
try:
|
||||||
loop = get_loop()
|
loop = get_loop()
|
||||||
if loop:
|
if loop:
|
||||||
|
|
Loading…
Reference in a new issue