diff --git a/searx/engines/bing.py b/searx/engines/bing.py
index 4c037de85..3d4ac08bd 100644
--- a/searx/engines/bing.py
+++ b/searx/engines/bing.py
@@ -8,7 +8,8 @@
import re
from urllib.parse import urlencode, urlparse, parse_qs
from lxml import html
-from searx.utils import eval_xpath, extract_text, match_language
+from searx.utils import eval_xpath, extract_text, eval_xpath_list, match_language
+from searx.network import multi_requests, Request
about = {
"website": 'https://www.bing.com',
@@ -79,30 +80,48 @@ def response(resp):
dom = html.fromstring(resp.text)
- for result in eval_xpath(dom, '//div[@class="sa_cc"]'):
-
- # IMO //div[@class="sa_cc"] does no longer match
- logger.debug('found //div[@class="sa_cc"] --> %s', result)
-
- link = eval_xpath(result, './/h3/a')[0]
- url = link.attrib.get('href')
- title = extract_text(link)
- content = extract_text(eval_xpath(result, './/p'))
-
- # append result
- results.append({'url': url, 'title': title, 'content': content})
-
# parse results again if nothing is found yet
- for result in eval_xpath(dom, '//li[@class="b_algo"]'):
+
+ url_to_resolve = []
+ url_to_resolve_index = []
+ for i, result in enumerate(eval_xpath_list(dom, '//li[@class="b_algo"]')):
link = eval_xpath(result, './/h2/a')[0]
url = link.attrib.get('href')
title = extract_text(link)
content = extract_text(eval_xpath(result, './/p'))
+ # get the real URL either using the URL shown to user or following the Bing URL
+ if url.startswith('https://www.bing.com/ck/a?'):
+ url_cite = extract_text(eval_xpath(result, './/div[@class="b_attribution"]/cite'))
+ # Bing can shorten the URL either at the end or in the middle of the string
+ if (
+ url_cite.startswith('https://')
+ and '…' not in url_cite
+ and '...' not in url_cite
+ and '›' not in url_cite
+ ):
+ # no need for an additional HTTP request
+ url = url_cite
+ else:
+ # resolve the URL with an additional HTTP request
+ url_to_resolve.append(url.replace('&ntb=1', '&ntb=F'))
+ url_to_resolve_index.append(i)
+ url = None # remove the result if the HTTP Bing redirect raise an exception
+
# append result
results.append({'url': url, 'title': title, 'content': content})
+ # resolve all Bing redirections in parallel
+ request_list = [
+ Request.get(u, allow_redirects=False, headers=resp.search_params['headers']) for u in url_to_resolve
+ ]
+ response_list = multi_requests(request_list)
+ for i, redirect_response in enumerate(response_list):
+ if not isinstance(redirect_response, Exception):
+ results[url_to_resolve_index[i]]['url'] = redirect_response.headers['location']
+
+ # get number_of_results
try:
result_len_container = "".join(eval_xpath(dom, '//span[@class="sb_count"]//text()'))
if "-" in result_len_container:
diff --git a/searx/network/__init__.py b/searx/network/__init__.py
index 06c9f75a4..8622e9731 100644
--- a/searx/network/__init__.py
+++ b/searx/network/__init__.py
@@ -8,7 +8,8 @@ import concurrent.futures
from queue import SimpleQueue
from types import MethodType
from timeit import default_timer
-from typing import Iterable, Tuple
+from typing import Iterable, NamedTuple, Tuple, List, Dict, Union
+from contextlib import contextmanager
import httpx
import anyio
@@ -48,9 +49,23 @@ def get_context_network():
return THREADLOCAL.__dict__.get('network') or get_network()
-def request(method, url, **kwargs):
- """same as requests/requests/api.py request(...)"""
+@contextmanager
+def _record_http_time():
+ # pylint: disable=too-many-branches
time_before_request = default_timer()
+ start_time = getattr(THREADLOCAL, 'start_time', time_before_request)
+ try:
+ yield start_time
+ finally:
+ # update total_time.
+ # See get_time_for_thread() and reset_time_for_thread()
+ if hasattr(THREADLOCAL, 'total_time'):
+ time_after_request = default_timer()
+ THREADLOCAL.total_time += time_after_request - time_before_request
+
+
+def _get_timeout(start_time, kwargs):
+ # pylint: disable=too-many-branches
# timeout (httpx)
if 'timeout' in kwargs:
@@ -65,45 +80,84 @@ def request(method, url, **kwargs):
# ajdust actual timeout
timeout += 0.2 # overhead
- start_time = getattr(THREADLOCAL, 'start_time', time_before_request)
if start_time:
timeout -= default_timer() - start_time
- # raise_for_error
- check_for_httperror = True
- if 'raise_for_httperror' in kwargs:
- check_for_httperror = kwargs['raise_for_httperror']
- del kwargs['raise_for_httperror']
+ return timeout
- # requests compatibility
- if isinstance(url, bytes):
- url = url.decode()
- # network
- network = get_context_network()
+def request(method, url, **kwargs):
+ """same as requests/requests/api.py request(...)"""
+ with _record_http_time() as start_time:
+ network = get_context_network()
+ timeout = _get_timeout(start_time, kwargs)
+ future = asyncio.run_coroutine_threadsafe(network.request(method, url, **kwargs), get_loop())
+ try:
+ return future.result(timeout)
+ except concurrent.futures.TimeoutError as e:
+ raise httpx.TimeoutException('Timeout', request=None) from e
- # do request
- future = asyncio.run_coroutine_threadsafe(network.request(method, url, **kwargs), get_loop())
- try:
- response = future.result(timeout)
- except concurrent.futures.TimeoutError as e:
- raise httpx.TimeoutException('Timeout', request=None) from e
- # requests compatibility
- # see also https://www.python-httpx.org/compatibility/#checking-for-4xx5xx-responses
- response.ok = not response.is_error
+def multi_requests(request_list: List["Request"]) -> List[Union[httpx.Response, Exception]]:
+ """send multiple HTTP requests in parallel. Wait for all requests to finish."""
+ with _record_http_time() as start_time:
+ # send the requests
+ network = get_context_network()
+ loop = get_loop()
+ future_list = []
+ for request_desc in request_list:
+ timeout = _get_timeout(start_time, request_desc.kwargs)
+ future = asyncio.run_coroutine_threadsafe(
+ network.request(request_desc.method, request_desc.url, **request_desc.kwargs), loop
+ )
+ future_list.append((future, timeout))
- # update total_time.
- # See get_time_for_thread() and reset_time_for_thread()
- if hasattr(THREADLOCAL, 'total_time'):
- time_after_request = default_timer()
- THREADLOCAL.total_time += time_after_request - time_before_request
+ # read the responses
+ responses = []
+ for future, timeout in future_list:
+ try:
+ responses.append(future.result(timeout))
+ except concurrent.futures.TimeoutError:
+ responses.append(httpx.TimeoutException('Timeout', request=None))
+ except Exception as e: # pylint: disable=broad-except
+ responses.append(e)
+ return responses
- # raise an exception
- if check_for_httperror:
- raise_for_httperror(response)
- return response
+class Request(NamedTuple):
+ """Request description for the multi_requests function"""
+
+ method: str
+ url: str
+ kwargs: Dict[str, str] = {}
+
+ @staticmethod
+ def get(url, **kwargs):
+ return Request('GET', url, kwargs)
+
+ @staticmethod
+ def options(url, **kwargs):
+ return Request('OPTIONS', url, kwargs)
+
+ @staticmethod
+ def head(url, **kwargs):
+ return Request('HEAD', url, kwargs)
+
+ @staticmethod
+ def post(url, **kwargs):
+ return Request('POST', url, kwargs)
+
+ @staticmethod
+ def put(url, **kwargs):
+ return Request('PUT', url, kwargs)
+
+ @staticmethod
+ def patch(url, **kwargs):
+ return Request('PATCH', url, kwargs)
+
+ @staticmethod
+ def delete(url, **kwargs):
+ return Request('DELETE', url, kwargs)
def get(url, **kwargs):
diff --git a/searx/network/network.py b/searx/network/network.py
index 69af3b7c4..677a908bf 100644
--- a/searx/network/network.py
+++ b/searx/network/network.py
@@ -13,6 +13,7 @@ import httpx
from searx import logger, searx_debug
from .client import new_client, get_loop, AsyncHTTPTransportNoHttp
+from .raise_for_httperror import raise_for_httperror
logger = logger.getChild('network')
@@ -226,6 +227,27 @@ class Network:
kwargs['follow_redirects'] = kwargs.pop('allow_redirects')
return kwargs_clients
+ @staticmethod
+ def extract_do_raise_for_httperror(kwargs):
+ do_raise_for_httperror = True
+ if 'raise_for_httperror' in kwargs:
+ do_raise_for_httperror = kwargs['raise_for_httperror']
+ del kwargs['raise_for_httperror']
+ return do_raise_for_httperror
+
+ @staticmethod
+ def patch_response(response, do_raise_for_httperror):
+ if isinstance(response, httpx.Response):
+ # requests compatibility (response is not streamed)
+ # see also https://www.python-httpx.org/compatibility/#checking-for-4xx5xx-responses
+ response.ok = not response.is_error
+
+ # raise an exception
+ if do_raise_for_httperror:
+ raise_for_httperror(response)
+
+ return response
+
def is_valid_response(self, response):
# pylint: disable=too-many-boolean-expressions
if (
@@ -239,6 +261,7 @@ class Network:
async def call_client(self, stream, method, url, **kwargs):
retries = self.retries
was_disconnected = False
+ do_raise_for_httperror = Network.extract_do_raise_for_httperror(kwargs)
kwargs_clients = Network.extract_kwargs_clients(kwargs)
while retries >= 0: # pragma: no cover
client = await self.get_client(**kwargs_clients)
@@ -248,7 +271,7 @@ class Network:
else:
response = await client.request(method, url, **kwargs)
if self.is_valid_response(response) or retries <= 0:
- return response
+ return Network.patch_response(response, do_raise_for_httperror)
except httpx.RemoteProtocolError as e:
if not was_disconnected:
# the server has closed the connection:
diff --git a/tests/unit/network/test_network.py b/tests/unit/network/test_network.py
index 4253e69ac..905b981c1 100644
--- a/tests/unit/network/test_network.py
+++ b/tests/unit/network/test_network.py
@@ -141,28 +141,28 @@ class TestNetworkRequestRetries(SearxTestCase):
async def test_retries_ok(self):
with patch.object(httpx.AsyncClient, 'request', new=TestNetworkRequestRetries.get_response_404_then_200()):
network = Network(enable_http=True, retries=1, retry_on_http_error=403)
- response = await network.request('GET', 'https://example.com/')
+ response = await network.request('GET', 'https://example.com/', raise_for_httperror=False)
self.assertEqual(response.text, TestNetworkRequestRetries.TEXT)
await network.aclose()
async def test_retries_fail_int(self):
with patch.object(httpx.AsyncClient, 'request', new=TestNetworkRequestRetries.get_response_404_then_200()):
network = Network(enable_http=True, retries=0, retry_on_http_error=403)
- response = await network.request('GET', 'https://example.com/')
+ response = await network.request('GET', 'https://example.com/', raise_for_httperror=False)
self.assertEqual(response.status_code, 403)
await network.aclose()
async def test_retries_fail_list(self):
with patch.object(httpx.AsyncClient, 'request', new=TestNetworkRequestRetries.get_response_404_then_200()):
network = Network(enable_http=True, retries=0, retry_on_http_error=[403, 429])
- response = await network.request('GET', 'https://example.com/')
+ response = await network.request('GET', 'https://example.com/', raise_for_httperror=False)
self.assertEqual(response.status_code, 403)
await network.aclose()
async def test_retries_fail_bool(self):
with patch.object(httpx.AsyncClient, 'request', new=TestNetworkRequestRetries.get_response_404_then_200()):
network = Network(enable_http=True, retries=0, retry_on_http_error=True)
- response = await network.request('GET', 'https://example.com/')
+ response = await network.request('GET', 'https://example.com/', raise_for_httperror=False)
self.assertEqual(response.status_code, 403)
await network.aclose()
@@ -178,7 +178,7 @@ class TestNetworkRequestRetries(SearxTestCase):
with patch.object(httpx.AsyncClient, 'request', new=get_response):
network = Network(enable_http=True, retries=2)
- response = await network.request('GET', 'https://example.com/')
+ response = await network.request('GET', 'https://example.com/', raise_for_httperror=False)
self.assertEqual(response.status_code, 200)
self.assertEqual(response.text, TestNetworkRequestRetries.TEXT)
await network.aclose()
@@ -190,7 +190,7 @@ class TestNetworkRequestRetries(SearxTestCase):
with patch.object(httpx.AsyncClient, 'request', new=get_response):
network = Network(enable_http=True, retries=0)
with self.assertRaises(httpx.RequestError):
- await network.request('GET', 'https://example.com/')
+ await network.request('GET', 'https://example.com/', raise_for_httperror=False)
await network.aclose()
@@ -237,6 +237,6 @@ class TestNetworkStreamRetries(SearxTestCase):
with patch.object(httpx.AsyncClient, 'stream', new=stream):
network = Network(enable_http=True, retries=0, retry_on_http_error=403)
- response = await network.stream('GET', 'https://example.com/')
+ response = await network.stream('GET', 'https://example.com/', raise_for_httperror=False)
self.assertEqual(response.status_code, 403)
await network.aclose()