forked from Ponysearch/Ponysearch
Adam Tauber
GitHub
commit
ceb8ae6439
3 changed files with 154 additions and 131 deletions
|
|
@ -90,6 +90,9 @@ def load_engine(engine_data):
|
|||
'result_count': 0,
|
||||
'search_count': 0,
|
||||
'page_load_time': 0,
|
||||
'page_load_count': 0,
|
||||
'engine_time': 0,
|
||||
'engine_time_count': 0,
|
||||
'score_count': 0,
|
||||
'errors': 0
|
||||
}
|
||||
|
|
@ -106,32 +109,56 @@ def load_engine(engine_data):
|
|||
return engine
|
||||
|
||||
|
||||
def to_percentage(stats, maxvalue):
|
||||
for engine_stat in stats:
|
||||
if maxvalue:
|
||||
engine_stat['percentage'] = int(engine_stat['avg'] / maxvalue * 100)
|
||||
else:
|
||||
engine_stat['percentage'] = 0
|
||||
return stats
|
||||
|
||||
|
||||
def get_engines_stats():
|
||||
# TODO refactor
|
||||
pageloads = []
|
||||
engine_times = []
|
||||
results = []
|
||||
scores = []
|
||||
errors = []
|
||||
scores_per_result = []
|
||||
|
||||
max_pageload = max_results = max_score = max_errors = max_score_per_result = 0 # noqa
|
||||
max_pageload = max_engine_times = max_results = max_score = max_errors = max_score_per_result = 0 # noqa
|
||||
for engine in engines.values():
|
||||
if engine.stats['search_count'] == 0:
|
||||
continue
|
||||
results_num = \
|
||||
engine.stats['result_count'] / float(engine.stats['search_count'])
|
||||
load_times = engine.stats['page_load_time'] / float(engine.stats['search_count']) # noqa
|
||||
|
||||
if engine.stats['page_load_count'] != 0:
|
||||
load_times = engine.stats['page_load_time'] / float(engine.stats['page_load_count']) # noqa
|
||||
else:
|
||||
load_times = 0
|
||||
|
||||
if engine.stats['engine_time_count'] != 0:
|
||||
this_engine_time = engine.stats['engine_time'] / float(engine.stats['engine_time_count']) # noqa
|
||||
else:
|
||||
this_engine_time = 0
|
||||
|
||||
if results_num:
|
||||
score = engine.stats['score_count'] / float(engine.stats['search_count']) # noqa
|
||||
score_per_result = score / results_num
|
||||
else:
|
||||
score = score_per_result = 0.0
|
||||
max_results = max(results_num, max_results)
|
||||
|
||||
max_pageload = max(load_times, max_pageload)
|
||||
max_engine_times = max(this_engine_time, max_engine_times)
|
||||
max_results = max(results_num, max_results)
|
||||
max_score = max(score, max_score)
|
||||
max_score_per_result = max(score_per_result, max_score_per_result)
|
||||
max_errors = max(max_errors, engine.stats['errors'])
|
||||
|
||||
pageloads.append({'avg': load_times, 'name': engine.name})
|
||||
engine_times.append({'avg': this_engine_time, 'name': engine.name})
|
||||
results.append({'avg': results_num, 'name': engine.name})
|
||||
scores.append({'avg': score, 'name': engine.name})
|
||||
errors.append({'avg': engine.stats['errors'], 'name': engine.name})
|
||||
|
|
@ -140,38 +167,18 @@ def get_engines_stats():
|
|||
'name': engine.name
|
||||
})
|
||||
|
||||
for engine in pageloads:
|
||||
if max_pageload:
|
||||
engine['percentage'] = int(engine['avg'] / max_pageload * 100)
|
||||
else:
|
||||
engine['percentage'] = 0
|
||||
|
||||
for engine in results:
|
||||
if max_results:
|
||||
engine['percentage'] = int(engine['avg'] / max_results * 100)
|
||||
else:
|
||||
engine['percentage'] = 0
|
||||
|
||||
for engine in scores:
|
||||
if max_score:
|
||||
engine['percentage'] = int(engine['avg'] / max_score * 100)
|
||||
else:
|
||||
engine['percentage'] = 0
|
||||
|
||||
for engine in scores_per_result:
|
||||
if max_score_per_result:
|
||||
engine['percentage'] = int(engine['avg']
|
||||
/ max_score_per_result * 100)
|
||||
else:
|
||||
engine['percentage'] = 0
|
||||
|
||||
for engine in errors:
|
||||
if max_errors:
|
||||
engine['percentage'] = int(float(engine['avg']) / max_errors * 100)
|
||||
else:
|
||||
engine['percentage'] = 0
|
||||
pageloads = to_percentage(pageloads, max_pageload)
|
||||
engine_times = to_percentage(engine_times, max_engine_times)
|
||||
results = to_percentage(results, max_results)
|
||||
scores = to_percentage(scores, max_score)
|
||||
scores_per_result = to_percentage(scores_per_result, max_score_per_result)
|
||||
erros = to_percentage(errors, max_errors)
|
||||
|
||||
return [
|
||||
(
|
||||
gettext('Engine time (sec)'),
|
||||
sorted(engine_times, key=itemgetter('avg'))
|
||||
),
|
||||
(
|
||||
gettext('Page loads (sec)'),
|
||||
sorted(pageloads, key=itemgetter('avg'))
|
||||
|
|
|
|||
206
searx/search.py
206
searx/search.py
|
|
@ -36,14 +36,53 @@ logger = logger.getChild('search')
|
|||
number_of_searches = 0
|
||||
|
||||
|
||||
def search_request_wrapper(fn, url, engine_name, **kwargs):
|
||||
ret = None
|
||||
engine = engines[engine_name]
|
||||
def send_http_request(engine, request_params, timeout_limit):
|
||||
response = None
|
||||
try:
|
||||
ret = fn(url, **kwargs)
|
||||
# create dictionary which contain all
|
||||
# informations about the request
|
||||
request_args = dict(
|
||||
headers=request_params['headers'],
|
||||
cookies=request_params['cookies'],
|
||||
timeout=timeout_limit,
|
||||
verify=request_params['verify']
|
||||
)
|
||||
# specific type of request (GET or POST)
|
||||
if request_params['method'] == 'GET':
|
||||
req = requests_lib.get
|
||||
else:
|
||||
req = requests_lib.post
|
||||
request_args['data'] = request_params['data']
|
||||
|
||||
# for page_load_time stats
|
||||
time_before_request = time()
|
||||
|
||||
# send the request
|
||||
response = req(request_params['url'], **request_args)
|
||||
|
||||
with threading.RLock():
|
||||
# no error : reset the suspend variables
|
||||
engine.continuous_errors = 0
|
||||
engine.suspend_end_time = 0
|
||||
# update stats with current page-load-time
|
||||
# only the HTTP request
|
||||
engine.stats['page_load_time'] += time() - time_before_request
|
||||
engine.stats['page_load_count'] += 1
|
||||
|
||||
# is there a timeout (no parsing in this case)
|
||||
timeout_overhead = 0.2 # seconds
|
||||
search_duration = time() - request_params['started']
|
||||
if search_duration > timeout_limit + timeout_overhead:
|
||||
logger.exception('engine timeout on HTTP request:'
|
||||
'{0} (search duration : {1} ms, time-out: {2} )'
|
||||
.format(engine.name, search_duration, timeout_limit))
|
||||
with threading.RLock():
|
||||
engine.stats['errors'] += 1
|
||||
return False
|
||||
|
||||
# everything is ok : return the response
|
||||
return response
|
||||
|
||||
except:
|
||||
# increase errors stats
|
||||
with threading.RLock():
|
||||
|
|
@ -52,20 +91,62 @@ def search_request_wrapper(fn, url, engine_name, **kwargs):
|
|||
engine.suspend_end_time = time() + min(60, engine.continuous_errors)
|
||||
|
||||
# print engine name and specific error message
|
||||
logger.exception('engine crash: {0}'.format(engine_name))
|
||||
return ret
|
||||
logger.exception('engine crash: {0}'.format(engine.name))
|
||||
return False
|
||||
|
||||
|
||||
def threaded_requests(requests):
|
||||
timeout_limit = max(r[2]['timeout'] for r in requests)
|
||||
search_start = time()
|
||||
def search_one_request(engine_name, query, request_params, result_container, timeout_limit):
|
||||
engine = engines[engine_name]
|
||||
|
||||
# update request parameters dependent on
|
||||
# search-engine (contained in engines folder)
|
||||
engine.request(query, request_params)
|
||||
|
||||
# TODO add support of offline engines
|
||||
if request_params['url'] is None:
|
||||
return False
|
||||
|
||||
# ignoring empty urls
|
||||
if not request_params['url']:
|
||||
return False
|
||||
|
||||
# send request
|
||||
response = send_http_request(engine, request_params, timeout_limit)
|
||||
|
||||
# parse response
|
||||
success = None
|
||||
if response:
|
||||
# parse the response
|
||||
response.search_params = request_params
|
||||
search_results = engine.response(response)
|
||||
|
||||
# add results
|
||||
for result in search_results:
|
||||
result['engine'] = engine.name
|
||||
|
||||
result_container.extend(engine.name, search_results)
|
||||
|
||||
success = True
|
||||
else:
|
||||
success = False
|
||||
|
||||
with threading.RLock():
|
||||
# update stats : total time
|
||||
engine.stats['engine_time'] += time() - request_params['started']
|
||||
engine.stats['engine_time_count'] += 1
|
||||
|
||||
#
|
||||
return success
|
||||
|
||||
|
||||
def search_multiple_requests(requests, result_container, timeout_limit):
|
||||
start_time = time()
|
||||
search_id = uuid4().__str__()
|
||||
for fn, url, request_args, engine_name in requests:
|
||||
request_args['timeout'] = timeout_limit
|
||||
|
||||
for engine_name, query, request_params in requests:
|
||||
th = threading.Thread(
|
||||
target=search_request_wrapper,
|
||||
args=(fn, url, engine_name),
|
||||
kwargs=request_args,
|
||||
target=search_one_request,
|
||||
args=(engine_name, query, request_params, result_container, timeout_limit),
|
||||
name=search_id,
|
||||
)
|
||||
th._engine_name = engine_name
|
||||
|
|
@ -73,7 +154,7 @@ def threaded_requests(requests):
|
|||
|
||||
for th in threading.enumerate():
|
||||
if th.name == search_id:
|
||||
remaining_time = max(0.0, timeout_limit - (time() - search_start))
|
||||
remaining_time = max(0.0, timeout_limit - (time() - start_time))
|
||||
th.join(remaining_time)
|
||||
if th.isAlive():
|
||||
logger.warning('engine timeout: {0}'.format(th._engine_name))
|
||||
|
|
@ -91,44 +172,6 @@ def default_request_params():
|
|||
}
|
||||
|
||||
|
||||
# create a callback wrapper for the search engine results
|
||||
def make_callback(engine_name, callback, params, result_container):
|
||||
|
||||
# creating a callback wrapper for the search engine results
|
||||
def process_callback(response, **kwargs):
|
||||
# check if redirect comparing to the True value,
|
||||
# because resp can be a Mock object, and any attribut name returns something.
|
||||
if response.is_redirect is True:
|
||||
logger.debug('{0} redirect on: {1}'.format(engine_name, response))
|
||||
return
|
||||
|
||||
response.search_params = params
|
||||
|
||||
search_duration = time() - params['started']
|
||||
# update stats with current page-load-time
|
||||
with threading.RLock():
|
||||
engines[engine_name].stats['page_load_time'] += search_duration
|
||||
|
||||
timeout_overhead = 0.2 # seconds
|
||||
timeout_limit = engines[engine_name].timeout + timeout_overhead
|
||||
|
||||
if search_duration > timeout_limit:
|
||||
with threading.RLock():
|
||||
engines[engine_name].stats['errors'] += 1
|
||||
return
|
||||
|
||||
# callback
|
||||
search_results = callback(response)
|
||||
|
||||
# add results
|
||||
for result in search_results:
|
||||
result['engine'] = engine_name
|
||||
|
||||
result_container.extend(engine_name, search_results)
|
||||
|
||||
return process_callback
|
||||
|
||||
|
||||
def get_search_query_from_webapp(preferences, form):
|
||||
query = None
|
||||
query_engines = []
|
||||
|
|
@ -255,6 +298,10 @@ class Search(object):
|
|||
def search(self):
|
||||
global number_of_searches
|
||||
|
||||
# start time
|
||||
start_time = time()
|
||||
|
||||
# answeres ?
|
||||
answerers_results = ask(self.search_query)
|
||||
|
||||
if answerers_results:
|
||||
|
|
@ -274,6 +321,9 @@ class Search(object):
|
|||
|
||||
search_query = self.search_query
|
||||
|
||||
# max of all selected engine timeout
|
||||
timeout_limit = 0
|
||||
|
||||
# start search-reqest for all selected engines
|
||||
for selected_engine in search_query.engines:
|
||||
if selected_engine['name'] not in engines:
|
||||
|
|
@ -303,7 +353,7 @@ class Search(object):
|
|||
request_params = default_request_params()
|
||||
request_params['headers']['User-Agent'] = user_agent
|
||||
request_params['category'] = selected_engine['category']
|
||||
request_params['started'] = time()
|
||||
request_params['started'] = start_time
|
||||
request_params['pageno'] = search_query.pageno
|
||||
|
||||
if hasattr(engine, 'language') and engine.language:
|
||||
|
|
@ -315,51 +365,15 @@ class Search(object):
|
|||
request_params['safesearch'] = search_query.safesearch
|
||||
request_params['time_range'] = search_query.time_range
|
||||
|
||||
# update request parameters dependent on
|
||||
# search-engine (contained in engines folder)
|
||||
engine.request(search_query.query.encode('utf-8'), request_params)
|
||||
|
||||
if request_params['url'] is None:
|
||||
# TODO add support of offline engines
|
||||
pass
|
||||
|
||||
# create a callback wrapper for the search engine results
|
||||
callback = make_callback(
|
||||
selected_engine['name'],
|
||||
engine.response,
|
||||
request_params,
|
||||
self.result_container)
|
||||
|
||||
# create dictionary which contain all
|
||||
# informations about the request
|
||||
request_args = dict(
|
||||
headers=request_params['headers'],
|
||||
hooks=dict(response=callback),
|
||||
cookies=request_params['cookies'],
|
||||
timeout=engine.timeout,
|
||||
verify=request_params['verify']
|
||||
)
|
||||
|
||||
# specific type of request (GET or POST)
|
||||
if request_params['method'] == 'GET':
|
||||
req = requests_lib.get
|
||||
else:
|
||||
req = requests_lib.post
|
||||
request_args['data'] = request_params['data']
|
||||
|
||||
# ignoring empty urls
|
||||
if not request_params['url']:
|
||||
continue
|
||||
|
||||
# append request to list
|
||||
requests.append((req, request_params['url'],
|
||||
request_args,
|
||||
selected_engine['name']))
|
||||
requests.append((selected_engine['name'], search_query.query.encode('utf-8'), request_params))
|
||||
|
||||
if not requests:
|
||||
return self.result_container
|
||||
# update timeout_limit
|
||||
timeout_limit = max(timeout_limit, engine.timeout)
|
||||
|
||||
if requests:
|
||||
# send all search-request
|
||||
threaded_requests(requests)
|
||||
search_multiple_requests(requests, self.result_container, timeout_limit - (time() - start_time))
|
||||
start_new_thread(gc.collect, tuple())
|
||||
|
||||
# return results, suggestions, answers and infoboxes
|
||||
|
|
|
|||
|
|
@ -601,6 +601,8 @@ def preferences():
|
|||
if e.timeout > settings['outgoing']['request_timeout']:
|
||||
stats[e.name]['warn_timeout'] = True
|
||||
|
||||
# get first element [0], the engine time,
|
||||
# and then the second element [1] : the time (the first one is the label)
|
||||
for engine_stat in get_engines_stats()[0][1]:
|
||||
stats[engine_stat.get('name')]['time'] = round(engine_stat.get('avg'), 3)
|
||||
if engine_stat.get('avg') > settings['outgoing']['request_timeout']:
|
||||
|
|
|
|||
Loading…
Reference in a new issue