From 16ff8d06c73d3e0ec784e8c9a1fb38b36a95d8fb Mon Sep 17 00:00:00 2001
From: Adam Tauber <asciimoo@gmail.com>
Date: Mon, 7 Nov 2016 22:30:20 +0100
Subject: [PATCH 01/28] [fix] bing paging and language support
see https://msdn.microsoft.com/en-us/library/ff795620.aspx for bing
specific search operators
closes #755
---
searx/engines/bing.py | 11 +++--------
tests/unit/engines/test_bing.py | 6 ++----
2 files changed, 5 insertions(+), 12 deletions(-)
diff --git a/searx/engines/bing.py b/searx/engines/bing.py
index 6bdfd378b..b9c5f73fc 100644
--- a/searx/engines/bing.py
+++ b/searx/engines/bing.py
@@ -32,18 +32,13 @@ search_string = 'search?{query}&first={offset}'
def request(query, params):
offset = (params['pageno'] - 1) * 10 + 1
- if params['language'] == 'all':
- language = 'en-US'
- else:
- language = params['language'].replace('_', '-')
+ if params['language'] != 'all':
+ query = 'language:{} {}'.format(params['language'].split('_')[0].upper(), query)
search_path = search_string.format(
- query=urlencode({'q': query, 'setmkt': language}),
+ query=urlencode({'q': query}),
offset=offset)
- params['cookies']['SRCHHPGUSR'] = \
- 'NEWWND=0&NRSLT=-1&SRCHLANG=' + language.split('-')[0]
-
params['url'] = base_url + search_path
return params
diff --git a/tests/unit/engines/test_bing.py b/tests/unit/engines/test_bing.py
index bce221440..886584229 100644
--- a/tests/unit/engines/test_bing.py
+++ b/tests/unit/engines/test_bing.py
@@ -14,14 +14,12 @@ class TestBingEngine(SearxTestCase):
params = bing.request(query, dicto)
self.assertTrue('url' in params)
self.assertTrue(query in params['url'])
+ self.assertTrue('language%3AFR' in params['url'])
self.assertTrue('bing.com' in params['url'])
- self.assertTrue('SRCHHPGUSR' in params['cookies'])
- self.assertTrue('fr' in params['cookies']['SRCHHPGUSR'])
dicto['language'] = 'all'
params = bing.request(query, dicto)
- self.assertTrue('SRCHHPGUSR' in params['cookies'])
- self.assertTrue('en' in params['cookies']['SRCHHPGUSR'])
+ self.assertTrue('language' not in params['url'])
def test_response(self):
self.assertRaises(AttributeError, bing.response, None)
From 17b08d096c5d7823799cdd6eab7fe67ef9941f9f Mon Sep 17 00:00:00 2001
From: Adam Tauber <asciimoo@gmail.com>
Date: Mon, 7 Nov 2016 22:33:17 +0100
Subject: [PATCH 02/28] [fix] unicode search expression for bing
---
searx/engines/bing.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/searx/engines/bing.py b/searx/engines/bing.py
index b9c5f73fc..75814a595 100644
--- a/searx/engines/bing.py
+++ b/searx/engines/bing.py
@@ -33,7 +33,7 @@ def request(query, params):
offset = (params['pageno'] - 1) * 10 + 1
if params['language'] != 'all':
- query = 'language:{} {}'.format(params['language'].split('_')[0].upper(), query)
+ query = u'language:{} {}'.format(params['language'].split('_')[0].upper(), query)
search_path = search_string.format(
query=urlencode({'q': query}),
From 1176505fa4d58677ca05e7a1c27ee459d86275aa Mon Sep 17 00:00:00 2001
From: Adam Tauber <asciimoo@gmail.com>
Date: Mon, 14 Nov 2016 15:47:42 +0100
Subject: [PATCH 03/28] [fix] bing character encoding - closes #760
---
searx/engines/bing.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/searx/engines/bing.py b/searx/engines/bing.py
index 75814a595..768cbdeb5 100644
--- a/searx/engines/bing.py
+++ b/searx/engines/bing.py
@@ -33,10 +33,10 @@ def request(query, params):
offset = (params['pageno'] - 1) * 10 + 1
if params['language'] != 'all':
- query = u'language:{} {}'.format(params['language'].split('_')[0].upper(), query)
+ query = u'language:{} {}'.format(params['language'].split('_')[0].upper(), query.decode('utf-8'))
search_path = search_string.format(
- query=urlencode({'q': query}),
+ query=urlencode({'q': query.encode('utf-8')}),
offset=offset)
params['url'] = base_url + search_path
From 94196c4b6c92da621b45db6332f594d59af52471 Mon Sep 17 00:00:00 2001
From: Adam Tauber <asciimoo@gmail.com>
Date: Mon, 14 Nov 2016 15:49:06 +0100
Subject: [PATCH 04/28] [enh] show traceback of search errors
---
searx/webapp.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/searx/webapp.py b/searx/webapp.py
index d3d5bb51e..7d5adaf07 100644
--- a/searx/webapp.py
+++ b/searx/webapp.py
@@ -410,7 +410,8 @@ def index():
# search = Search(search_query) # without plugins
search = SearchWithPlugins(search_query, request)
result_container = search.search()
- except:
+ except Exception:
+ logger.exception('search error')
return render(
'index.html',
)
From 16f2e346b3b5507034f6207e5d829cceaa99c046 Mon Sep 17 00:00:00 2001
From: Adam Tauber <asciimoo@gmail.com>
Date: Mon, 14 Nov 2016 15:52:29 +0100
Subject: [PATCH 05/28] [fix] bing unicode issue part III.
---
searx/engines/bing.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/searx/engines/bing.py b/searx/engines/bing.py
index 768cbdeb5..24cefaffa 100644
--- a/searx/engines/bing.py
+++ b/searx/engines/bing.py
@@ -33,10 +33,10 @@ def request(query, params):
offset = (params['pageno'] - 1) * 10 + 1
if params['language'] != 'all':
- query = u'language:{} {}'.format(params['language'].split('_')[0].upper(), query.decode('utf-8'))
+ query = u'language:{} {}'.format(params['language'].split('_')[0].upper(), query.decode('utf-8')).encode('utf-8')
search_path = search_string.format(
- query=urlencode({'q': query.encode('utf-8')}),
+ query=urlencode({'q': query}),
offset=offset)
params['url'] = base_url + search_path
From 43ddbc60da114e3c764e6f7fa344e686964b66c2 Mon Sep 17 00:00:00 2001
From: Adam Tauber <asciimoo@gmail.com>
Date: Mon, 14 Nov 2016 16:09:16 +0100
Subject: [PATCH 06/28] [fix] pep8
---
searx/engines/bing.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/searx/engines/bing.py b/searx/engines/bing.py
index 24cefaffa..540597162 100644
--- a/searx/engines/bing.py
+++ b/searx/engines/bing.py
@@ -33,7 +33,8 @@ def request(query, params):
offset = (params['pageno'] - 1) * 10 + 1
if params['language'] != 'all':
- query = u'language:{} {}'.format(params['language'].split('_')[0].upper(), query.decode('utf-8')).encode('utf-8')
+ query = u'language:{} {}'.format(params['language'].split('_')[0].upper(),
+ query.decode('utf-8')).encode('utf-8')
search_path = search_string.format(
query=urlencode({'q': query}),
From 96f182d75d76a77457137abd89b3cc374c26aa9f Mon Sep 17 00:00:00 2001
From: Adam Tauber <asciimoo@gmail.com>
Date: Mon, 14 Nov 2016 16:17:11 +0100
Subject: [PATCH 07/28] [fix] allow empty autocomplete setting
closes #756
closes #761
---
searx/preferences.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/searx/preferences.py b/searx/preferences.py
index ed7b6f6cd..8b787fc59 100644
--- a/searx/preferences.py
+++ b/searx/preferences.py
@@ -218,7 +218,7 @@ class Preferences(object):
'locale': EnumStringSetting(settings['ui']['default_locale'],
choices=settings['locales'].keys()),
'autocomplete': EnumStringSetting(settings['search']['autocomplete'],
- choices=autocomplete.backends.keys()),
+ choices=autocomplete.backends.keys() + ['']),
'image_proxy': MapSetting(settings['server']['image_proxy'],
map={'': settings['server']['image_proxy'],
'0': False,
From 88dfee858e93e54ad6e54801f88b93bfdc2bb149 Mon Sep 17 00:00:00 2001
From: Adam Tauber <asciimoo@gmail.com>
Date: Mon, 14 Nov 2016 17:25:21 +0100
Subject: [PATCH 08/28] [fix] rewrite missing variable
---
searx/plugins/doai_rewrite.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/searx/plugins/doai_rewrite.py b/searx/plugins/doai_rewrite.py
index 0142af672..a6e15ae5a 100644
--- a/searx/plugins/doai_rewrite.py
+++ b/searx/plugins/doai_rewrite.py
@@ -27,5 +27,5 @@ def on_result(request, search, result):
if doi.endswith(suffix):
doi = doi[:-len(suffix)]
result['url'] = 'http://doai.io/' + doi
- result['parsed_url'] = urlparse(ctx['result']['url'])
+ result['parsed_url'] = urlparse(result['url'])
return True
From 832cf37a97b97061a26e2fdf49c293d26e917ef5 Mon Sep 17 00:00:00 2001
From: Adam Tauber <asciimoo@gmail.com>
Date: Mon, 14 Nov 2016 22:07:23 +0100
Subject: [PATCH 09/28] [enh] display errors
also tried flask's flash feature but flask creates session cookies if it
isn't flushed. Avoiding session cookies to preserve privacy
---
searx/templates/oscar/base.html | 15 +++++++++++++++
searx/webapp.py | 16 ++++++++++------
2 files changed, 25 insertions(+), 6 deletions(-)
diff --git a/searx/templates/oscar/base.html b/searx/templates/oscar/base.html
index 81d4febcf..220f5f8b1 100644
--- a/searx/templates/oscar/base.html
+++ b/searx/templates/oscar/base.html
@@ -1,3 +1,4 @@
+{% from 'oscar/macros.html' import icon %}
<!DOCTYPE html>
<html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en"{% if rtl %} dir="rtl"{% endif %}>
<head>
@@ -54,6 +55,20 @@
<body>
{% include 'oscar/navbar.html' %}
<div class="container">
+ {% if errors %}
+ <div class="alert alert-danger fade in" role="alert">
+ <button class="close" data-dismiss="alert" type="button">
+ <span aria-hidden="true">×</span>
+ <span class="sr-only">{{ _('Close') }}</span>
+ </button>
+ <strong class="lead">{{ icon('info-sign') }} {{ _('Error!') }}</strong>
+ <ul>
+ {% for message in errors %}
+ <li>{{ message }}</li>
+ {% endfor %}
+ </ul>
+ </div>
+ {% endif %}
{% block site_alert_error %}
{% endblock %}
diff --git a/searx/webapp.py b/searx/webapp.py
index 7d5adaf07..68902a6cd 100644
--- a/searx/webapp.py
+++ b/searx/webapp.py
@@ -344,6 +344,8 @@ def render(template_name, override_theme=None, **kwargs):
kwargs['cookies'] = request.cookies
+ kwargs['errors'] = request.errors
+
kwargs['instance_name'] = settings['general']['instance_name']
kwargs['results_on_new_tab'] = request.preferences.get_value('results_on_new_tab')
@@ -364,15 +366,16 @@ def render(template_name, override_theme=None, **kwargs):
@app.before_request
def pre_request():
- # merge GET, POST vars
+ request.errors = []
+
preferences = Preferences(themes, categories.keys(), engines, plugins)
try:
preferences.parse_cookies(request.cookies)
except:
- # TODO throw error message to the user
- logger.warning('Invalid config')
+ request.errors.append(gettext('Invalid settings, please edit your preferences'))
request.preferences = preferences
+ # merge GET, POST vars
# request.form
request.form = dict(request.form.items())
for k, v in request.args.items():
@@ -397,7 +400,7 @@ def index():
Supported outputs: html, json, csv, rss.
"""
- if not request.args and not request.form:
+ if request.form.get('q') is None:
return render(
'index.html',
)
@@ -410,7 +413,8 @@ def index():
# search = Search(search_query) # without plugins
search = SearchWithPlugins(search_query, request)
result_container = search.search()
- except Exception:
+ except:
+ request.errors.append(gettext('search error'))
logger.exception('search error')
return render(
'index.html',
@@ -573,7 +577,7 @@ def preferences():
try:
request.preferences.parse_form(request.form)
except ValidationException:
- # TODO use flash feature of flask
+ request.errors.append(gettext('Invalid settings, please edit your preferences'))
return resp
return request.preferences.save(resp)
From a757c2f005f38a97d7ebe53ab7271e26c13b7197 Mon Sep 17 00:00:00 2001
From: Adam Tauber <asciimoo@gmail.com>
Date: Mon, 14 Nov 2016 22:15:03 +0100
Subject: [PATCH 10/28] [fix] remove unused imports
---
searx/webapp.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/searx/webapp.py b/searx/webapp.py
index 68902a6cd..45332ba99 100644
--- a/searx/webapp.py
+++ b/searx/webapp.py
@@ -62,8 +62,8 @@ from searx.utils import (
)
from searx.version import VERSION_STRING
from searx.languages import language_codes
-from searx.search import Search, SearchWithPlugins, get_search_query_from_webapp
-from searx.query import RawTextQuery, SearchQuery
+from searx.search import SearchWithPlugins, get_search_query_from_webapp
+from searx.query import RawTextQuery
from searx.autocomplete import searx_bang, backends as autocomplete_backends
from searx.plugins import plugins
from searx.preferences import Preferences, ValidationException
From 044809e2980193f6947d26da84ef66d03321c4b1 Mon Sep 17 00:00:00 2001
From: Adam Tauber <asciimoo@gmail.com>
Date: Mon, 14 Nov 2016 22:21:19 +0100
Subject: [PATCH 11/28] [fix] search mocking in webapp test
---
tests/unit/test_webapp.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/tests/unit/test_webapp.py b/tests/unit/test_webapp.py
index 912bebc4d..1ef1f56c3 100644
--- a/tests/unit/test_webapp.py
+++ b/tests/unit/test_webapp.py
@@ -5,6 +5,7 @@ from mock import Mock
from urlparse import ParseResult
from searx import webapp
from searx.testing import SearxTestCase
+from searx.search import Search
class ViewsTestCase(SearxTestCase):
@@ -41,7 +42,7 @@ class ViewsTestCase(SearxTestCase):
results_number=lambda: 3,
results_length=lambda: len(self.test_results))
- webapp.Search.search = search_mock
+ Search.search = search_mock
def get_current_theme_name_mock(override=None):
return 'legacy'
From 12c369e858b45b4904ab079c3aefebaa18e7ece4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= <sitbackandwait@gmail.com>
Date: Mon, 14 Nov 2016 22:24:40 +0100
Subject: [PATCH 12/28] preferences: refactor to check consistently input
values
---
searx/preferences.py | 26 ++++++++++++++------------
1 file changed, 14 insertions(+), 12 deletions(-)
diff --git a/searx/preferences.py b/searx/preferences.py
index 8b787fc59..045f0e8c6 100644
--- a/searx/preferences.py
+++ b/searx/preferences.py
@@ -49,28 +49,32 @@ class StringSetting(Setting):
class EnumStringSetting(Setting):
"""Setting of a value which can only come from the given choices"""
+ def _validate_selection(self, selection):
+ if selection not in self.choices:
+ raise ValidationException('Invalid value: "{0}"'.format(selection))
+
def _post_init(self):
if not hasattr(self, 'choices'):
raise MissingArgumentException('Missing argument: choices')
-
- if self.value != '' and self.value not in self.choices:
- raise ValidationException('Invalid default value: {0}'.format(self.value))
+ self._validate_selection(self.value)
def parse(self, data):
- if data not in self.choices and data != self.value:
- raise ValidationException('Invalid choice: {0}'.format(data))
+ self._validate_selection(data)
self.value = data
class MultipleChoiceSetting(EnumStringSetting):
"""Setting of values which can only come from the given choices"""
+ def _validate_selections(self, selections):
+ for item in selections:
+ if item not in self.choices:
+ raise ValidationException('Invalid value: "{0}"'.format(selections))
+
def _post_init(self):
if not hasattr(self, 'choices'):
raise MissingArgumentException('Missing argument: choices')
- for item in self.value:
- if item not in self.choices:
- raise ValidationException('Invalid default value: {0}'.format(self.value))
+ self._validate_selections(self.value)
def parse(self, data):
if data == '':
@@ -78,9 +82,7 @@ class MultipleChoiceSetting(EnumStringSetting):
return
elements = data.split(',')
- for item in elements:
- if item not in self.choices:
- raise ValidationException('Invalid choice: {0}'.format(item))
+ self._validate_selections(elements)
self.value = elements
def parse_form(self, data):
@@ -216,7 +218,7 @@ class Preferences(object):
self.key_value_settings = {'categories': MultipleChoiceSetting(['general'], choices=categories),
'language': EnumStringSetting('all', choices=LANGUAGE_CODES),
'locale': EnumStringSetting(settings['ui']['default_locale'],
- choices=settings['locales'].keys()),
+ choices=settings['locales'].keys() + ['']),
'autocomplete': EnumStringSetting(settings['search']['autocomplete'],
choices=autocomplete.backends.keys() + ['']),
'image_proxy': MapSetting(settings['server']['image_proxy'],
From 299c8823045a269c83d19cb6d05b24ef334207af Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= <sitbackandwait@gmail.com>
Date: Mon, 14 Nov 2016 22:32:40 +0100
Subject: [PATCH 13/28] search: make language configurable from settings.yml
---
searx/preferences.py | 3 ++-
searx/settings.yml | 1 +
searx/settings_robot.yml | 1 +
3 files changed, 4 insertions(+), 1 deletion(-)
diff --git a/searx/preferences.py b/searx/preferences.py
index 045f0e8c6..4436b8fe8 100644
--- a/searx/preferences.py
+++ b/searx/preferences.py
@@ -216,7 +216,8 @@ class Preferences(object):
super(Preferences, self).__init__()
self.key_value_settings = {'categories': MultipleChoiceSetting(['general'], choices=categories),
- 'language': EnumStringSetting('all', choices=LANGUAGE_CODES),
+ 'language': EnumStringSetting(settings['search']['language'],
+ choices=LANGUAGE_CODES),
'locale': EnumStringSetting(settings['ui']['default_locale'],
choices=settings['locales'].keys() + ['']),
'autocomplete': EnumStringSetting(settings['search']['autocomplete'],
diff --git a/searx/settings.yml b/searx/settings.yml
index 573cf5458..733341c31 100644
--- a/searx/settings.yml
+++ b/searx/settings.yml
@@ -5,6 +5,7 @@ general:
search:
safe_search : 0 # Filter results. 0: None, 1: Moderate, 2: Strict
autocomplete : "" # Existing autocomplete backends: "dbpedia", "duckduckgo", "google", "startpage", "wikipedia" - leave blank to turn it off by default
+ language : "all"
server:
port : 8888
diff --git a/searx/settings_robot.yml b/searx/settings_robot.yml
index 43dc9b00a..7d2701449 100644
--- a/searx/settings_robot.yml
+++ b/searx/settings_robot.yml
@@ -5,6 +5,7 @@ general:
search:
safe_search : 0
autocomplete : ""
+ language: "all"
server:
port : 11111
From 827f9e41ca84638d873997001b53e7f4a62a78aa Mon Sep 17 00:00:00 2001
From: Adam Tauber <asciimoo@gmail.com>
Date: Tue, 15 Nov 2016 09:56:18 +0100
Subject: [PATCH 14/28] [fix] gettext requires request.preferences
---
searx/webapp.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/searx/webapp.py b/searx/webapp.py
index 45332ba99..090df57e2 100644
--- a/searx/webapp.py
+++ b/searx/webapp.py
@@ -369,11 +369,11 @@ def pre_request():
request.errors = []
preferences = Preferences(themes, categories.keys(), engines, plugins)
+ request.preferences = preferences
try:
preferences.parse_cookies(request.cookies)
except:
request.errors.append(gettext('Invalid settings, please edit your preferences'))
- request.preferences = preferences
# merge GET, POST vars
# request.form
From 55dc538398090e437c5e495dddad983a7870d09b Mon Sep 17 00:00:00 2001
From: Adam Tauber <asciimoo@gmail.com>
Date: Sat, 19 Nov 2016 17:51:19 +0100
Subject: [PATCH 15/28] [mod] move load_module function to utils
---
searx/engines/__init__.py | 17 ++++-------------
searx/utils.py | 13 +++++++++++++
2 files changed, 17 insertions(+), 13 deletions(-)
diff --git a/searx/engines/__init__.py b/searx/engines/__init__.py
index 14376c31f..116eadc97 100644
--- a/searx/engines/__init__.py
+++ b/searx/engines/__init__.py
@@ -16,13 +16,13 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
(C) 2013- by Adam Tauber, <asciimoo@gmail.com>
'''
-from os.path import realpath, dirname, splitext, join
+from os.path import realpath, dirname
import sys
-from imp import load_source
from flask_babel import gettext
from operator import itemgetter
from searx import settings
from searx import logger
+from searx.utils import load_module
logger = logger.getChild('engines')
@@ -32,6 +32,7 @@ engine_dir = dirname(realpath(__file__))
engines = {}
categories = {'general': []}
+_initialized = False
engine_shortcuts = {}
engine_default_args = {'paging': False,
@@ -46,16 +47,6 @@ engine_default_args = {'paging': False,
'time_range_support': False}
-def load_module(filename):
- modname = splitext(filename)[0]
- if modname in sys.modules:
- del sys.modules[modname]
- filepath = join(engine_dir, filename)
- module = load_source(modname, filepath)
- module.name = modname
- return module
-
-
def load_engine(engine_data):
if '_' in engine_data['name']:
@@ -65,7 +56,7 @@ def load_engine(engine_data):
engine_module = engine_data['engine']
try:
- engine = load_module(engine_module + '.py')
+ engine = load_module(engine_module + '.py', engine_dir)
except:
logger.exception('Cannot load engine "{}"'.format(engine_module))
return None
diff --git a/searx/utils.py b/searx/utils.py
index 5039fa975..faa634853 100644
--- a/searx/utils.py
+++ b/searx/utils.py
@@ -6,7 +6,10 @@ import re
from babel.dates import format_date
from codecs import getincrementalencoder
from HTMLParser import HTMLParser
+from imp import load_source
+from os.path import splitext, join
from random import choice
+import sys
from searx.version import VERSION_STRING
from searx.languages import language_codes
@@ -285,3 +288,13 @@ def is_valid_lang(lang):
if l[1].lower() == lang.lower():
return (True, l[0][:2], l[1].lower())
return False
+
+
+def load_module(filename, module_dir):
+ modname = splitext(filename)[0]
+ if modname in sys.modules:
+ del sys.modules[modname]
+ filepath = join(module_dir, filename)
+ module = load_source(modname, filepath)
+ module.name = modname
+ return module
From 971ed0abd159625bd01d0b3ca52c90b394711d77 Mon Sep 17 00:00:00 2001
From: Adam Tauber <asciimoo@gmail.com>
Date: Sat, 19 Nov 2016 20:53:51 +0100
Subject: [PATCH 16/28] [enh] add quick answer functionality with an example
answerer
---
searx/answerers/__init__.py | 46 ++++++++++++++++++++++++
searx/answerers/random/answerer.py | 50 ++++++++++++++++++++++++++
searx/results.py | 9 ++---
searx/search.py | 8 +++++
searx/templates/oscar/preferences.html | 29 +++++++++++++++
searx/webapp.py | 2 ++
tests/unit/test_answerers.py | 16 +++++++++
7 files changed, 156 insertions(+), 4 deletions(-)
create mode 100644 searx/answerers/__init__.py
create mode 100644 searx/answerers/random/answerer.py
create mode 100644 tests/unit/test_answerers.py
diff --git a/searx/answerers/__init__.py b/searx/answerers/__init__.py
new file mode 100644
index 000000000..8f5951c75
--- /dev/null
+++ b/searx/answerers/__init__.py
@@ -0,0 +1,46 @@
+from os import listdir
+from os.path import realpath, dirname, join, isdir
+from searx.utils import load_module
+from collections import defaultdict
+
+
+answerers_dir = dirname(realpath(__file__))
+
+
+def load_answerers():
+ answerers = []
+ for filename in listdir(answerers_dir):
+ if not isdir(join(answerers_dir, filename)):
+ continue
+ module = load_module('answerer.py', join(answerers_dir, filename))
+ if not hasattr(module, 'keywords') or not isinstance(module.keywords, tuple) or not len(module.keywords):
+ exit(2)
+ answerers.append(module)
+ return answerers
+
+
+def get_answerers_by_keywords(answerers):
+ by_keyword = defaultdict(list)
+ for answerer in answerers:
+ for keyword in answerer.keywords:
+ for keyword in answerer.keywords:
+ by_keyword[keyword].append(answerer.answer)
+ return by_keyword
+
+
+def ask(query):
+ results = []
+ query_parts = filter(None, query.query.split())
+
+ if query_parts[0] not in answerers_by_keywords:
+ return results
+
+ for answerer in answerers_by_keywords[query_parts[0]]:
+ result = answerer(query)
+ if result:
+ results.append(result)
+ return results
+
+
+answerers = load_answerers()
+answerers_by_keywords = get_answerers_by_keywords(answerers)
diff --git a/searx/answerers/random/answerer.py b/searx/answerers/random/answerer.py
new file mode 100644
index 000000000..510d9f5be
--- /dev/null
+++ b/searx/answerers/random/answerer.py
@@ -0,0 +1,50 @@
+import random
+import string
+from flask_babel import gettext
+
+# required answerer attribute
+# specifies which search query keywords triggers this answerer
+keywords = ('random',)
+
+random_int_max = 2**31
+
+random_string_letters = string.lowercase + string.digits + string.uppercase
+
+
+def random_string():
+ return u''.join(random.choice(random_string_letters)
+ for _ in range(random.randint(8, 32)))
+
+
+def random_float():
+ return unicode(random.random())
+
+
+def random_int():
+ return unicode(random.randint(-random_int_max, random_int_max))
+
+
+random_types = {u'string': random_string,
+ u'int': random_int,
+ u'float': random_float}
+
+
+# required answerer function
+# can return a list of results (any result type) for a given query
+def answer(query):
+ parts = query.query.split()
+ if len(parts) != 2:
+ return []
+
+ if parts[1] not in random_types:
+ return []
+
+ return [{'answer': random_types[parts[1]]()}]
+
+
+# required answerer function
+# returns information about the answerer
+def self_info():
+ return {'name': gettext('Random value generator'),
+ 'description': gettext('Generate different random values'),
+ 'examples': [u'random {}'.format(x) for x in random_types]}
diff --git a/searx/results.py b/searx/results.py
index 634f71acd..73a96c081 100644
--- a/searx/results.py
+++ b/searx/results.py
@@ -146,16 +146,17 @@ class ResultContainer(object):
self._number_of_results.append(result['number_of_results'])
results.remove(result)
- with RLock():
- engines[engine_name].stats['search_count'] += 1
- engines[engine_name].stats['result_count'] += len(results)
+ if engine_name in engines:
+ with RLock():
+ engines[engine_name].stats['search_count'] += 1
+ engines[engine_name].stats['result_count'] += len(results)
if not results:
return
self.results[engine_name].extend(results)
- if not self.paging and engines[engine_name].paging:
+ if not self.paging and engine_name in engines and engines[engine_name].paging:
self.paging = True
for i, result in enumerate(results):
diff --git a/searx/search.py b/searx/search.py
index c3f1566a9..0095de821 100644
--- a/searx/search.py
+++ b/searx/search.py
@@ -24,6 +24,7 @@ import searx.poolrequests as requests_lib
from searx.engines import (
categories, engines
)
+from searx.answerers import ask
from searx.utils import gen_useragent
from searx.query import RawTextQuery, SearchQuery
from searx.results import ResultContainer
@@ -254,6 +255,13 @@ class Search(object):
def search(self):
global number_of_searches
+ answerers_results = ask(self.search_query)
+
+ if answerers_results:
+ for results in answerers_results:
+ self.result_container.extend('answer', results)
+ return self.result_container
+
# init vars
requests = []
diff --git a/searx/templates/oscar/preferences.html b/searx/templates/oscar/preferences.html
index ed790c56a..6ad795095 100644
--- a/searx/templates/oscar/preferences.html
+++ b/searx/templates/oscar/preferences.html
@@ -12,6 +12,7 @@
<li class="active"><a href="#tab_general" role="tab" data-toggle="tab">{{ _('General') }}</a></li>
<li><a href="#tab_engine" role="tab" data-toggle="tab">{{ _('Engines') }}</a></li>
<li><a href="#tab_plugins" role="tab" data-toggle="tab">{{ _('Plugins') }}</a></li>
+ {% if answerers %}<li><a href="#tab_answerers" role="tab" data-toggle="tab">{{ _('Answerers') }}</a></li>{% endif %}
<li><a href="#tab_cookies" role="tab" data-toggle="tab">{{ _('Cookies') }}</a></li>
</ul>
@@ -224,6 +225,34 @@
</fieldset>
</div>
+ {% if answerers %}
+ <div class="tab-pane active_if_nojs" id="tab_answerers">
+ <noscript>
+ <h3>{{ _('Answerers') }}</h3>
+ </noscript>
+ <p class="text-muted" style="margin:20px 0;">
+ {{ _('This is the list of searx\'s instant answering modules.') }}
+ </p>
+ <table class="table table-striped">
+ <tr>
+ <th class="text-muted">{{ _('Name') }}</th>
+ <th class="text-muted">{{ _('Keywords') }}</th>
+ <th class="text-muted">{{ _('Description') }}</th>
+ <th class="text-muted">{{ _('Examples') }}</th>
+ </tr>
+
+ {% for answerer in answerers %}
+ <tr>
+ <td class="text-muted">{{ answerer.info.name }}</td>
+ <td class="text-muted">{{ answerer.keywords|join(', ') }}</td>
+ <td class="text-muted">{{ answerer.info.description }}</td>
+ <td class="text-muted">{{ answerer.info.examples|join(', ') }}</td>
+ </tr>
+ {% endfor %}
+ </table>
+ </div>
+ {% endif %}
+
<div class="tab-pane active_if_nojs" id="tab_cookies">
<noscript>
<h3>{{ _('Cookies') }}</h3>
diff --git a/searx/webapp.py b/searx/webapp.py
index 090df57e2..8996aa2b9 100644
--- a/searx/webapp.py
+++ b/searx/webapp.py
@@ -67,6 +67,7 @@ from searx.query import RawTextQuery
from searx.autocomplete import searx_bang, backends as autocomplete_backends
from searx.plugins import plugins
from searx.preferences import Preferences, ValidationException
+from searx.answerers import answerers
# check if the pyopenssl, ndg-httpsclient, pyasn1 packages are installed.
# They are needed for SSL connection without trouble, see #298
@@ -612,6 +613,7 @@ def preferences():
language_codes=language_codes,
engines_by_category=categories,
stats=stats,
+ answerers=[{'info': a.self_info(), 'keywords': a.keywords} for a in answerers],
disabled_engines=disabled_engines,
autocomplete_backends=autocomplete_backends,
shortcuts={y: x for x, y in engine_shortcuts.items()},
diff --git a/tests/unit/test_answerers.py b/tests/unit/test_answerers.py
new file mode 100644
index 000000000..bd8789a7e
--- /dev/null
+++ b/tests/unit/test_answerers.py
@@ -0,0 +1,16 @@
+# -*- coding: utf-8 -*-
+
+from mock import Mock
+
+from searx.answerers import answerers
+from searx.testing import SearxTestCase
+
+
+class AnswererTest(SearxTestCase):
+
+ def test_unicode_input(self):
+ query = Mock()
+ unicode_payload = u'árvíztűrő tükörfúrógép'
+ for answerer in answerers:
+ query.query = u'{} {}'.format(answerer.keywords[0], unicode_payload)
+ self.assertTrue(isinstance(answerer.answer(query), list))
From 7eed8a5dd909ca70d45fdff5706bb207e6aa8bdf Mon Sep 17 00:00:00 2001
From: Adam Tauber <asciimoo@gmail.com>
Date: Sat, 19 Nov 2016 21:03:27 +0100
Subject: [PATCH 17/28] [enh] add statistics answerer
---
searx/answerers/statistics/answerer.py | 51 ++++++++++++++++++++++++++
1 file changed, 51 insertions(+)
create mode 100644 searx/answerers/statistics/answerer.py
diff --git a/searx/answerers/statistics/answerer.py b/searx/answerers/statistics/answerer.py
new file mode 100644
index 000000000..3fef69c60
--- /dev/null
+++ b/searx/answerers/statistics/answerer.py
@@ -0,0 +1,51 @@
+from functools import reduce
+from operator import mul
+
+from flask_babel import gettext
+
+keywords = ('min',
+ 'max',
+ 'avg',
+ 'sum',
+ 'prod')
+
+
+# required answerer function
+# can return a list of results (any result type) for a given query
+def answer(query):
+ parts = query.query.split()
+
+ if len(parts) < 2:
+ return []
+
+ try:
+ args = map(float, parts[1:])
+ except:
+ return []
+
+ func = parts[0]
+ answer = None
+
+ if func == 'min':
+ answer = min(args)
+ elif func == 'max':
+ answer = max(args)
+ elif func == 'avg':
+ answer = sum(args)/len(args)
+ elif func == 'sum':
+ answer = sum(args)
+ elif func == 'prod':
+ answer = reduce(mul, args, 1)
+
+ if answer is None:
+ return []
+
+ return [{'answer': unicode(answer)}]
+
+
+# required answerer function
+# returns information about the answerer
+def self_info():
+ return {'name': gettext('Statistics functions'),
+ 'description': gettext('Compute {functions} of the arguments').format(functions='/'.join(keywords)),
+ 'examples': ['avg 123 548 2.04 24.2']}
From 0724bd8168789be42964b9cb5cdd030d62f0d2d1 Mon Sep 17 00:00:00 2001
From: Adam Tauber <asciimoo@gmail.com>
Date: Sat, 19 Nov 2016 21:18:12 +0100
Subject: [PATCH 18/28] [fix] pep8
---
searx/answerers/statistics/answerer.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/searx/answerers/statistics/answerer.py b/searx/answerers/statistics/answerer.py
index 3fef69c60..a04695f56 100644
--- a/searx/answerers/statistics/answerer.py
+++ b/searx/answerers/statistics/answerer.py
@@ -31,7 +31,7 @@ def answer(query):
elif func == 'max':
answer = max(args)
elif func == 'avg':
- answer = sum(args)/len(args)
+ answer = sum(args) / len(args)
elif func == 'sum':
answer = sum(args)
elif func == 'prod':
From 7986d4cf4192df645fc29fe6df12607bb6949bd9 Mon Sep 17 00:00:00 2001
From: Adam Tauber <asciimoo@gmail.com>
Date: Sun, 4 Dec 2016 11:17:17 +0100
Subject: [PATCH 19/28] [fix] correct path for autoscroll on non-root urls
closes #758
---
searx/static/plugins/js/infinite_scroll.js | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/searx/static/plugins/js/infinite_scroll.js b/searx/static/plugins/js/infinite_scroll.js
index 213f74b15..9cd582d7f 100644
--- a/searx/static/plugins/js/infinite_scroll.js
+++ b/searx/static/plugins/js/infinite_scroll.js
@@ -5,7 +5,7 @@ $(document).ready(function() {
var formData = $('#pagination form:last').serialize();
if (formData) {
$('#pagination').html('<div class="loading-spinner"></div>');
- $.post('/', formData, function (data) {
+ $.post('./', formData, function (data) {
var body = $(data);
$('#pagination').remove();
$('#main_results').append('<hr/>');
From 28f12ef5a0917b8cefddb4d5f74c9aaeb945355f Mon Sep 17 00:00:00 2001
From: Adam Tauber <asciimoo@gmail.com>
Date: Sun, 4 Dec 2016 23:07:46 +0100
Subject: [PATCH 20/28] [fix] proper escaping of the search query in templates
---
searx/templates/courgette/results.html | 10 +++++-----
searx/templates/legacy/results.html | 10 +++++-----
searx/templates/oscar/results.html | 14 +++++++-------
searx/templates/pix-art/results.html | 6 +++---
4 files changed, 20 insertions(+), 20 deletions(-)
diff --git a/searx/templates/courgette/results.html b/searx/templates/courgette/results.html
index 3ffbd5882..c72b7c3f7 100644
--- a/searx/templates/courgette/results.html
+++ b/searx/templates/courgette/results.html
@@ -1,6 +1,6 @@
{% extends "courgette/base.html" %}
-{% block title %}{{ q }} - {% endblock %}
-{% block meta %}<link rel="alternate" type="application/rss+xml" title="Searx search: {{ q }}" href="{{ url_for('index') }}?q={{ q|urlencode }}&format=rss&{% for category in selected_categories %}category_{{ category }}=1&{% endfor %}pageno={{ pageno }}">{% endblock %}
+{% block title %}{{ q|e }} - {% endblock %}
+{% block meta %}<link rel="alternate" type="application/rss+xml" title="Searx search: {{ q|e }}" href="{{ url_for('index') }}?q={{ q|urlencode }}&format=rss&{% for category in selected_categories %}category_{{ category }}=1&{% endfor %}pageno={{ pageno }}">{% endblock %}
{% block content %}
<div class="right"><a href="{{ url_for('preferences') }}" id="preferences"><span>{{ _('preferences') }}</span></a></div>
<div class="small search center">
@@ -17,7 +17,7 @@
{% for output_type in ('csv', 'json', 'rss') %}
<form method="{{ method or 'POST' }}" action="{{ url_for('index') }}">
<div class="left">
- <input type="hidden" name="q" value="{{ q }}" />
+ <input type="hidden" name="q" value="{{ q|e }}" />
<input type="hidden" name="format" value="{{ output_type }}" />
{% for category in selected_categories %}
<input type="hidden" name="category_{{ category }}" value="1"/>
@@ -62,7 +62,7 @@
{% if pageno > 1 %}
<form method="{{ method or 'POST' }}" action="{{ url_for('index') }}">
<div class="left">
- <input type="hidden" name="q" value="{{ q }}" />
+ <input type="hidden" name="q" value="{{ q|e }}" />
{% for category in selected_categories %}
<input type="hidden" name="category_{{ category }}" value="1"/>
{% endfor %}
@@ -76,7 +76,7 @@
{% for category in selected_categories %}
<input type="hidden" name="category_{{ category }}" value="1"/>
{% endfor %}
- <input type="hidden" name="q" value="{{ q }}" />
+ <input type="hidden" name="q" value="{{ q|e }}" />
<input type="hidden" name="pageno" value="{{ pageno+1 }}" />
<input type="submit" value="{{ _('next page') }} >>" />
</div>
diff --git a/searx/templates/legacy/results.html b/searx/templates/legacy/results.html
index f50700c6f..f0d78398d 100644
--- a/searx/templates/legacy/results.html
+++ b/searx/templates/legacy/results.html
@@ -1,6 +1,6 @@
{% extends "legacy/base.html" %}
-{% block title %}{{ q }} - {% endblock %}
-{% block meta %}<link rel="alternate" type="application/rss+xml" title="Searx search: {{ q }}" href="{{ url_for('index') }}?q={{ q|urlencode }}&format=rss&{% for category in selected_categories %}category_{{ category }}=1&{% endfor %}pageno={{ pageno }}">{% endblock %}
+{% block title %}{{ q|e }} - {% endblock %}
+{% block meta %}<link rel="alternate" type="application/rss+xml" title="Searx search: {{ q|e }}" href="{{ url_for('index') }}?q={{ q|urlencode }}&format=rss&{% for category in selected_categories %}category_{{ category }}=1&{% endfor %}pageno={{ pageno }}">{% endblock %}
{% block content %}
<div class="preferences_container right"><a href="{{ url_for('preferences') }}" id="preferences"><span>preferences</span></a></div>
<div class="small search center">
@@ -18,7 +18,7 @@
{% for output_type in ('csv', 'json', 'rss') %}
<form method="{{ method or 'POST' }}" action="{{ url_for('index') }}">
<div class="left">
- <input type="hidden" name="q" value="{{ q }}" />
+ <input type="hidden" name="q" value="{{ q|e }}" />
<input type="hidden" name="format" value="{{ output_type }}" />
{% for category in selected_categories %}
<input type="hidden" name="category_{{ category }}" value="1"/>
@@ -73,7 +73,7 @@
{% if pageno > 1 %}
<form method="{{ method or 'POST' }}" action="{{ url_for('index') }}">
<div class="{% if rtl %}right{% else %}left{% endif %}">
- <input type="hidden" name="q" value="{{ q }}" />
+ <input type="hidden" name="q" value="{{ q|e }}" />
{% for category in selected_categories %}
<input type="hidden" name="category_{{ category }}" value="1"/>
{% endfor %}
@@ -87,7 +87,7 @@
{% for category in selected_categories %}
<input type="hidden" name="category_{{ category }}" value="1"/>
{% endfor %}
- <input type="hidden" name="q" value="{{ q }}" />
+ <input type="hidden" name="q" value="{{ q|e }}" />
<input type="hidden" name="pageno" value="{{ pageno+1 }}" />
<input type="submit" value="{{ _('next page') }} >>" />
</div>
diff --git a/searx/templates/oscar/results.html b/searx/templates/oscar/results.html
index e71be325a..0ae83e74b 100644
--- a/searx/templates/oscar/results.html
+++ b/searx/templates/oscar/results.html
@@ -1,6 +1,6 @@
{% extends "oscar/base.html" %}
-{% block title %}{{ q }} - {% endblock %}
-{% block meta %}<link rel="alternate" type="application/rss+xml" title="Searx search: {{ q }}" href="{{ url_for('index') }}?q={{ q|urlencode }}&format=rss&{% for category in selected_categories %}category_{{ category }}=1&{% endfor %}pageno={{ pageno }}&time_range={{ time_range }}">{% endblock %}
+{% block title %}{{ q|e }} - {% endblock %}
+{% block meta %}<link rel="alternate" type="application/rss+xml" title="Searx search: {{ q|e }}" href="{{ url_for('index') }}?q={{ q|urlencode }}&format=rss&{% for category in selected_categories %}category_{{ category }}=1&{% endfor %}pageno={{ pageno }}&time_range={{ time_range }}">{% endblock %}
{% block content %}
<div class="row">
<div class="col-sm-8" id="main_results">
@@ -37,9 +37,9 @@
<div id="pagination">
<div class="pull-left">
<form method="{{ method or 'POST' }}" action="{{ url_for('index') }}" class="pull-left">
- <input type="hidden" name="q" value="{{ q }}" />
+ <input type="hidden" name="q" value="{{ q|e }}" />
{% for category in selected_categories %}<input type="hidden" name="category_{{ category }}" value="1"/>{% endfor %}
- <input type="hidden" name="q" value="{{ q }}" />
+ <input type="hidden" name="q" value="{{ q|e }}" />
<input type="hidden" name="pageno" value="{{ pageno+1 }}" />
<input type="hidden" name="time_range" value="{{ time_range }}" />
<button type="submit" class="btn btn-default"><span class="glyphicon glyphicon-backward"></span> {{ _('next page') }}</button>
@@ -59,7 +59,7 @@
<div id="pagination">
<div class="pull-left">
<form method="{{ method or 'POST' }}" action="{{ url_for('index') }}" class="pull-left">
- <input type="hidden" name="q" value="{{ q }}" />
+ <input type="hidden" name="q" value="{{ q|e }}" />
{% for category in selected_categories %}<input type="hidden" name="category_{{ category }}" value="1"/>{% endfor %}
<input type="hidden" name="pageno" value="{{ pageno-1 }}" />
<input type="hidden" name="time_range" value="{{ time_range }}" />
@@ -69,7 +69,7 @@
<div class="pull-right">
<form method="{{ method or 'POST' }}" action="{{ url_for('index') }}" class="pull-left">
{% for category in selected_categories %}<input type="hidden" name="category_{{ category }}" value="1"/>{% endfor %}
- <input type="hidden" name="q" value="{{ q }}" />
+ <input type="hidden" name="q" value="{{ q|e }}" />
<input type="hidden" name="pageno" value="{{ pageno+1 }}" />
<input type="hidden" name="time_range" value="{{ time_range }}" />
<button type="submit" class="btn btn-default"><span class="glyphicon glyphicon-forward"></span> {{ _('next page') }}</button>
@@ -130,7 +130,7 @@
<div class="clearfix"></div>
{% for output_type in ('csv', 'json', 'rss') %}
<form method="{{ method or 'POST' }}" action="{{ url_for('index') }}" class="form-inline pull-{% if rtl %}right{% else %}left{% endif %} result_download">
- <input type="hidden" name="q" value="{{ q }}">
+ <input type="hidden" name="q" value="{{ q|e }}">
<input type="hidden" name="format" value="{{ output_type }}">
{% for category in selected_categories %}<input type="hidden" name="category_{{ category }}" value="1">{% endfor %}
<input type="hidden" name="pageno" value="{{ pageno }}">
diff --git a/searx/templates/pix-art/results.html b/searx/templates/pix-art/results.html
index f7d0e209b..8999e0513 100644
--- a/searx/templates/pix-art/results.html
+++ b/searx/templates/pix-art/results.html
@@ -5,7 +5,7 @@
{% endfor %}
{% else %}
{% extends "pix-art/base.html" %}
-{% block title %}{{ q }} - {% endblock %}
+{% block title %}{{ q|e }} - {% endblock %}
{% block meta %}{% endblock %}
{% block content %}
<div id="logo"><a href="./"><img src="{{ url_for('static', filename='img/searx-pixel-small.png') }}" alt="searx Logo"/></a></div>
@@ -25,8 +25,8 @@
</span>
<div id="pagination">
<br />
- <input type="button" onclick="load_more('{{ q }}', {{ pageno+1 }})" id="load_more" value="{{ _('Load more...') }}" />
+ <input type="button" onclick="load_more('{{ q|e }}', {{ pageno+1 }})" id="load_more" value="{{ _('Load more...') }}" />
</div>
</div>
{% endblock %}
-{% endif %}
\ No newline at end of file
+{% endif %}
From 16bdc0baf4f2b56af000337c4a2fa1e689f1220c Mon Sep 17 00:00:00 2001
From: Adam Tauber <asciimoo@gmail.com>
Date: Fri, 9 Dec 2016 11:44:24 +0100
Subject: [PATCH 21/28] [mod] do not escape html content in engines
---
searx/engines/archlinux.py | 3 +--
searx/engines/base.py | 3 +--
searx/engines/bing.py | 5 ++---
searx/engines/btdigg.py | 5 ++---
searx/engines/dailymotion.py | 3 +--
searx/engines/deezer.py | 9 +++++----
searx/engines/dictzone.py | 5 ++---
searx/engines/digg.py | 3 +--
searx/engines/fdroid.py | 3 +--
searx/engines/flickr.py | 14 +++-----------
searx/engines/flickr_noapi.py | 7 +++----
searx/engines/gigablast.py | 5 ++---
searx/engines/github.py | 3 +--
searx/engines/google.py | 5 ++---
searx/engines/kickass.py | 3 +--
searx/engines/nyaa.py | 6 ++----
searx/engines/piratebay.py | 3 +--
searx/engines/reddit.py | 3 +--
searx/engines/searchcode_doc.py | 12 ++----------
searx/engines/seedpeer.py | 1 -
searx/engines/spotify.py | 9 +++++----
searx/engines/stackoverflow.py | 5 ++---
searx/engines/startpage.py | 5 ++---
searx/engines/subtitleseeker.py | 5 ++---
searx/engines/swisscows.py | 9 ++++-----
searx/engines/tokyotoshokan.py | 1 -
searx/engines/torrentz.py | 1 -
searx/engines/translated.py | 11 +++++------
searx/engines/wolframalpha_noapi.py | 1 -
searx/engines/yandex.py | 5 ++---
30 files changed, 56 insertions(+), 97 deletions(-)
diff --git a/searx/engines/archlinux.py b/searx/engines/archlinux.py
index b846934f7..5ba512766 100644
--- a/searx/engines/archlinux.py
+++ b/searx/engines/archlinux.py
@@ -12,7 +12,6 @@
"""
from urlparse import urljoin
-from cgi import escape
from urllib import urlencode
from lxml import html
from searx.engines.xpath import extract_text
@@ -135,7 +134,7 @@ def response(resp):
for result in dom.xpath(xpath_results):
link = result.xpath(xpath_link)[0]
href = urljoin(base_url, link.attrib.get('href'))
- title = escape(extract_text(link))
+ title = extract_text(link)
results.append({'url': href,
'title': title})
diff --git a/searx/engines/base.py b/searx/engines/base.py
index 66491d395..a552453ce 100755
--- a/searx/engines/base.py
+++ b/searx/engines/base.py
@@ -16,7 +16,6 @@
from lxml import etree
from urllib import urlencode
from searx.utils import searx_useragent
-from cgi import escape
from datetime import datetime
import re
@@ -94,7 +93,7 @@ def response(resp):
url = item.text
elif item.attrib["name"] == "dcdescription":
- content = escape(item.text[:300])
+ content = item.text[:300]
if len(item.text) > 300:
content += "..."
diff --git a/searx/engines/bing.py b/searx/engines/bing.py
index 540597162..58db61251 100644
--- a/searx/engines/bing.py
+++ b/searx/engines/bing.py
@@ -14,7 +14,6 @@
"""
from urllib import urlencode
-from cgi import escape
from lxml import html
from searx.engines.xpath import extract_text
@@ -61,7 +60,7 @@ def response(resp):
link = result.xpath('.//h3/a')[0]
url = link.attrib.get('href')
title = extract_text(link)
- content = escape(extract_text(result.xpath('.//p')))
+ content = extract_text(result.xpath('.//p'))
# append result
results.append({'url': url,
@@ -73,7 +72,7 @@ def response(resp):
link = result.xpath('.//h2/a')[0]
url = link.attrib.get('href')
title = extract_text(link)
- content = escape(extract_text(result.xpath('.//p')))
+ content = extract_text(result.xpath('.//p'))
# append result
results.append({'url': url,
diff --git a/searx/engines/btdigg.py b/searx/engines/btdigg.py
index ea6baf1c8..33c8355de 100644
--- a/searx/engines/btdigg.py
+++ b/searx/engines/btdigg.py
@@ -11,7 +11,6 @@
"""
from urlparse import urljoin
-from cgi import escape
from urllib import quote
from lxml import html
from operator import itemgetter
@@ -51,8 +50,8 @@ def response(resp):
for result in search_res:
link = result.xpath('.//td[@class="torrent_name"]//a')[0]
href = urljoin(url, link.attrib.get('href'))
- title = escape(extract_text(link))
- content = escape(extract_text(result.xpath('.//pre[@class="snippet"]')[0]))
+ title = extract_text(link)
+ content = extract_text(result.xpath('.//pre[@class="snippet"]')[0])
content = "<br />".join(content.split("\n"))
filesize = result.xpath('.//span[@class="attr_val"]/text()')[0].split()[0]
diff --git a/searx/engines/dailymotion.py b/searx/engines/dailymotion.py
index 4eb894725..317f34f59 100644
--- a/searx/engines/dailymotion.py
+++ b/searx/engines/dailymotion.py
@@ -14,7 +14,6 @@
from urllib import urlencode
from json import loads
-from cgi import escape
from datetime import datetime
# engine dependent config
@@ -57,7 +56,7 @@ def response(resp):
for res in search_res['list']:
title = res['title']
url = res['url']
- content = escape(res['description'])
+ content = res['description']
thumbnail = res['thumbnail_360_url']
publishedDate = datetime.fromtimestamp(res['created_time'], None)
embedded = embedded_url.format(videoid=res['id'])
diff --git a/searx/engines/deezer.py b/searx/engines/deezer.py
index 0530bc072..8e87bbeec 100644
--- a/searx/engines/deezer.py
+++ b/searx/engines/deezer.py
@@ -51,10 +51,11 @@ def response(resp):
if url.startswith('http://'):
url = 'https' + url[4:]
- content = result['artist']['name'] +\
- " • " +\
- result['album']['title'] +\
- " • " + result['title']
+ content = '{} - {} - {}'.format(
+ result['artist']['name'],
+ result['album']['title'],
+ result['title'])
+
embedded = embedded_url.format(audioid=result['id'])
# append result
diff --git a/searx/engines/dictzone.py b/searx/engines/dictzone.py
index 9765d5f60..20a9a8980 100644
--- a/searx/engines/dictzone.py
+++ b/searx/engines/dictzone.py
@@ -12,7 +12,6 @@
import re
from urlparse import urljoin
from lxml import html
-from cgi import escape
from searx.utils import is_valid_lang
categories = ['general']
@@ -62,8 +61,8 @@ def response(resp):
results.append({
'url': urljoin(resp.url, '?%d' % k),
- 'title': escape(from_result.text_content()),
- 'content': escape('; '.join(to_results))
+ 'title': from_result.text_content(),
+ 'content': '; '.join(to_results)
})
return results
diff --git a/searx/engines/digg.py b/searx/engines/digg.py
index a10b38bb6..238b466a0 100644
--- a/searx/engines/digg.py
+++ b/searx/engines/digg.py
@@ -13,7 +13,6 @@
from urllib import quote_plus
from json import loads
from lxml import html
-from cgi import escape
from dateutil import parser
# engine dependent config
@@ -56,7 +55,7 @@ def response(resp):
url = result.attrib.get('data-contenturl')
thumbnail = result.xpath('.//img')[0].attrib.get('src')
title = ''.join(result.xpath(title_xpath))
- content = escape(''.join(result.xpath(content_xpath)))
+ content = ''.join(result.xpath(content_xpath))
pubdate = result.xpath(pubdate_xpath)[0].attrib.get('datetime')
publishedDate = parser.parse(pubdate)
diff --git a/searx/engines/fdroid.py b/searx/engines/fdroid.py
index 0b16773e3..6d470a4eb 100644
--- a/searx/engines/fdroid.py
+++ b/searx/engines/fdroid.py
@@ -9,7 +9,6 @@
@parse url, title, content
"""
-from cgi import escape
from urllib import urlencode
from searx.engines.xpath import extract_text
from lxml import html
@@ -43,7 +42,7 @@ def response(resp):
img_src = app.xpath('.//img/@src')[0]
content = extract_text(app.xpath('./p')[0])
- content = escape(content.replace(title, '', 1).strip())
+ content = content.replace(title, '', 1).strip()
results.append({'url': url,
'title': title,
diff --git a/searx/engines/flickr.py b/searx/engines/flickr.py
index 68d45bc17..1c3eef789 100644
--- a/searx/engines/flickr.py
+++ b/searx/engines/flickr.py
@@ -77,21 +77,13 @@ def response(resp):
url = build_flickr_url(photo['owner'], photo['id'])
- title = photo['title']
-
- content = '<span class="photo-author">' +\
- photo['ownername'] +\
- '</span><br />' +\
- '<span class="description">' +\
- photo['description']['_content'] +\
- '</span>'
-
# append result
results.append({'url': url,
- 'title': title,
+ 'title': photo['title'],
'img_src': img_src,
'thumbnail_src': thumbnail_src,
- 'content': content,
+ 'content': content = photo['description']['_content'],
+ 'author': photo['ownername'],
'template': 'images.html'})
# return results
diff --git a/searx/engines/flickr_noapi.py b/searx/engines/flickr_noapi.py
index 5c4193c11..68be139be 100644
--- a/searx/engines/flickr_noapi.py
+++ b/searx/engines/flickr_noapi.py
@@ -102,16 +102,15 @@ def response(resp):
title = photo.get('title', '')
- content = '<span class="photo-author">' +\
- photo['username'] +\
- '</span><br />'
+ author = photo['username']
# append result
results.append({'url': url,
'title': title,
'img_src': img_src,
'thumbnail_src': thumbnail_src,
- 'content': content,
+ 'content': '',
+ 'author': author,
'template': 'images.html'})
return results
diff --git a/searx/engines/gigablast.py b/searx/engines/gigablast.py
index 6e4e24b68..5430eb3ba 100644
--- a/searx/engines/gigablast.py
+++ b/searx/engines/gigablast.py
@@ -10,7 +10,6 @@
@parse url, title, content
"""
-from cgi import escape
from json import loads
from random import randint
from time import time
@@ -78,8 +77,8 @@ def response(resp):
for result in response_json['results']:
# append result
results.append({'url': result['url'],
- 'title': escape(result['title']),
- 'content': escape(result['sum'])})
+ 'title': result['title'],
+ 'content': result['sum']})
# return results
return results
diff --git a/searx/engines/github.py b/searx/engines/github.py
index cc1fc470c..7adef3be9 100644
--- a/searx/engines/github.py
+++ b/searx/engines/github.py
@@ -12,7 +12,6 @@
from urllib import urlencode
from json import loads
-from cgi import escape
# engine dependent config
categories = ['it']
@@ -48,7 +47,7 @@ def response(resp):
url = res['html_url']
if res['description']:
- content = escape(res['description'][:500])
+ content = res['description'][:500]
else:
content = ''
diff --git a/searx/engines/google.py b/searx/engines/google.py
index ea93bc94f..0e2d522f4 100644
--- a/searx/engines/google.py
+++ b/searx/engines/google.py
@@ -9,7 +9,6 @@
# @parse url, title, content, suggestion
import re
-from cgi import escape
from urllib import urlencode
from urlparse import urlparse, parse_qsl
from lxml import html, etree
@@ -155,7 +154,7 @@ def parse_url(url_string, google_hostname):
def extract_text_from_dom(result, xpath):
r = result.xpath(xpath)
if len(r) > 0:
- return escape(extract_text(r[0]))
+ return extract_text(r[0])
return None
@@ -264,7 +263,7 @@ def response(resp):
# parse suggestion
for suggestion in dom.xpath(suggestion_xpath):
# append suggestion
- results.append({'suggestion': escape(extract_text(suggestion))})
+ results.append({'suggestion': extract_text(suggestion)})
# return results
return results
diff --git a/searx/engines/kickass.py b/searx/engines/kickass.py
index 9cd8284da..059fa2a66 100644
--- a/searx/engines/kickass.py
+++ b/searx/engines/kickass.py
@@ -11,7 +11,6 @@
"""
from urlparse import urljoin
-from cgi import escape
from urllib import quote
from lxml import html
from operator import itemgetter
@@ -57,7 +56,7 @@ def response(resp):
link = result.xpath('.//a[@class="cellMainLink"]')[0]
href = urljoin(url, link.attrib['href'])
title = extract_text(link)
- content = escape(extract_text(result.xpath(content_xpath)))
+ content = extract_text(result.xpath(content_xpath))
seed = extract_text(result.xpath('.//td[contains(@class, "green")]'))
leech = extract_text(result.xpath('.//td[contains(@class, "red")]'))
filesize_info = extract_text(result.xpath('.//td[contains(@class, "nobr")]'))
diff --git a/searx/engines/nyaa.py b/searx/engines/nyaa.py
index cda8231f7..4ca5b3171 100644
--- a/searx/engines/nyaa.py
+++ b/searx/engines/nyaa.py
@@ -9,7 +9,6 @@
@parse url, title, content, seed, leech, torrentfile
"""
-from cgi import escape
from urllib import urlencode
from lxml import html
from searx.engines.xpath import extract_text
@@ -78,7 +77,7 @@ def response(resp):
# torrent title
page_a = result.xpath(xpath_title)[0]
- title = escape(extract_text(page_a))
+ title = extract_text(page_a)
# link to the page
href = page_a.attrib.get('href')
@@ -90,7 +89,7 @@ def response(resp):
try:
file_size, suffix = result.xpath(xpath_filesize)[0].split(' ')
file_size = int(float(file_size) * get_filesize_mul(suffix))
- except Exception as e:
+ except:
file_size = None
# seed count
@@ -105,7 +104,6 @@ def response(resp):
# content string contains all information not included into template
content = 'Category: "{category}". Downloaded {downloads} times.'
content = content.format(category=category, downloads=downloads)
- content = escape(content)
results.append({'url': href,
'title': title,
diff --git a/searx/engines/piratebay.py b/searx/engines/piratebay.py
index 55446b410..ca21a3bb2 100644
--- a/searx/engines/piratebay.py
+++ b/searx/engines/piratebay.py
@@ -9,7 +9,6 @@
# @parse url, title, content, seed, leech, magnetlink
from urlparse import urljoin
-from cgi import escape
from urllib import quote
from lxml import html
from operator import itemgetter
@@ -62,7 +61,7 @@ def response(resp):
link = result.xpath('.//div[@class="detName"]//a')[0]
href = urljoin(url, link.attrib.get('href'))
title = extract_text(link)
- content = escape(extract_text(result.xpath(content_xpath)))
+ content = extract_text(result.xpath(content_xpath))
seed, leech = result.xpath('.//td[@align="right"]/text()')[:2]
# convert seed to int if possible
diff --git a/searx/engines/reddit.py b/searx/engines/reddit.py
index 3ca7e44f6..b29792a3a 100644
--- a/searx/engines/reddit.py
+++ b/searx/engines/reddit.py
@@ -11,7 +11,6 @@
"""
import json
-from cgi import escape
from urllib import urlencode
from urlparse import urlparse, urljoin
from datetime import datetime
@@ -68,7 +67,7 @@ def response(resp):
img_results.append(params)
else:
created = datetime.fromtimestamp(data['created_utc'])
- content = escape(data['selftext'])
+ content = data['selftext']
if len(content) > 500:
content = content[:500] + '...'
params['content'] = content
diff --git a/searx/engines/searchcode_doc.py b/searx/engines/searchcode_doc.py
index f24fe6f90..6c1acdcdd 100644
--- a/searx/engines/searchcode_doc.py
+++ b/searx/engines/searchcode_doc.py
@@ -44,20 +44,12 @@ def response(resp):
# parse results
for result in search_results.get('results', []):
href = result['url']
- title = "[" + result['type'] + "] " +\
- result['namespace'] +\
- " " + result['name']
- content = '<span class="highlight">[' +\
- result['type'] + "] " +\
- result['name'] + " " +\
- result['synopsis'] +\
- "</span><br />" +\
- result['description']
+ title = "[{}] {} {}".format(result['type'], result['namespace'], result['name'])
# append result
results.append({'url': href,
'title': title,
- 'content': content})
+ 'content': result['description']})
# return results
return results
diff --git a/searx/engines/seedpeer.py b/searx/engines/seedpeer.py
index 854ebba03..e1309a9b5 100644
--- a/searx/engines/seedpeer.py
+++ b/searx/engines/seedpeer.py
@@ -9,7 +9,6 @@
# @parse url, title, content, seed, leech, magnetlink
from urlparse import urljoin
-from cgi import escape
from urllib import quote
from lxml import html
from operator import itemgetter
diff --git a/searx/engines/spotify.py b/searx/engines/spotify.py
index f75796e83..0e8e69961 100644
--- a/searx/engines/spotify.py
+++ b/searx/engines/spotify.py
@@ -46,10 +46,11 @@ def response(resp):
if result['type'] == 'track':
title = result['name']
url = result['external_urls']['spotify']
- content = result['artists'][0]['name'] +\
- " • " +\
- result['album']['name'] +\
- " • " + result['name']
+ content = '{} - {} - {}'.format(
+ result['artists'][0]['name'],
+ result['album']['name'],
+ result['name'])
+
embedded = embedded_url.format(audioid=result['id'])
# append result
diff --git a/searx/engines/stackoverflow.py b/searx/engines/stackoverflow.py
index fdd3711a9..5e7ab2901 100644
--- a/searx/engines/stackoverflow.py
+++ b/searx/engines/stackoverflow.py
@@ -11,7 +11,6 @@
"""
from urlparse import urljoin
-from cgi import escape
from urllib import urlencode
from lxml import html
from searx.engines.xpath import extract_text
@@ -48,8 +47,8 @@ def response(resp):
for result in dom.xpath(results_xpath):
link = result.xpath(link_xpath)[0]
href = urljoin(url, link.attrib.get('href'))
- title = escape(extract_text(link))
- content = escape(extract_text(result.xpath(content_xpath)))
+ title = extract_text(link)
+ content = extract_text(result.xpath(content_xpath))
# append result
results.append({'url': href,
diff --git a/searx/engines/startpage.py b/searx/engines/startpage.py
index d8b702c4d..6f6eae1cf 100644
--- a/searx/engines/startpage.py
+++ b/searx/engines/startpage.py
@@ -11,7 +11,6 @@
# @todo paging
from lxml import html
-from cgi import escape
from dateutil import parser
from datetime import datetime, timedelta
import re
@@ -79,10 +78,10 @@ def response(resp):
if re.match(r"^http(s|)://(www\.)?ixquick\.com/do/search\?.*$", url):
continue
- title = escape(extract_text(link))
+ title = extract_text(link)
if result.xpath('./p[@class="desc clk"]'):
- content = escape(extract_text(result.xpath('./p[@class="desc clk"]')))
+ content = extract_text(result.xpath('./p[@class="desc clk"]'))
else:
content = ''
diff --git a/searx/engines/subtitleseeker.py b/searx/engines/subtitleseeker.py
index 47d27d0b2..daba68be7 100644
--- a/searx/engines/subtitleseeker.py
+++ b/searx/engines/subtitleseeker.py
@@ -10,7 +10,6 @@
@parse url, title, content
"""
-from cgi import escape
from urllib import quote_plus
from lxml import html
from searx.languages import language_codes
@@ -59,7 +58,7 @@ def response(resp):
elif search_lang:
href = href + search_lang + '/'
- title = escape(extract_text(link))
+ title = extract_text(link)
content = extract_text(result.xpath('.//div[contains(@class,"red")]'))
content = content + " - "
@@ -75,7 +74,7 @@ def response(resp):
# append result
results.append({'url': href,
'title': title,
- 'content': escape(content)})
+ 'content': content})
# return results
return results
diff --git a/searx/engines/swisscows.py b/searx/engines/swisscows.py
index 1a94ed64e..72184e428 100644
--- a/searx/engines/swisscows.py
+++ b/searx/engines/swisscows.py
@@ -10,7 +10,6 @@
@parse url, title, content
"""
-from cgi import escape
from json import loads
from urllib import urlencode, unquote
import re
@@ -78,7 +77,7 @@ def response(resp):
# append result
results.append({'url': result['SourceUrl'],
- 'title': escape(result['Title']),
+ 'title': result['Title'],
'content': '',
'img_src': img_url,
'template': 'images.html'})
@@ -90,8 +89,8 @@ def response(resp):
# append result
results.append({'url': result_url,
- 'title': escape(result_title),
- 'content': escape(result_content)})
+ 'title': result_title,
+ 'content': result_content})
# parse images
for result in json.get('Images', []):
@@ -100,7 +99,7 @@ def response(resp):
# append result
results.append({'url': result['SourceUrl'],
- 'title': escape(result['Title']),
+ 'title': result['Title'],
'content': '',
'img_src': img_url,
'template': 'images.html'})
diff --git a/searx/engines/tokyotoshokan.py b/searx/engines/tokyotoshokan.py
index e2990e153..52b2cbe07 100644
--- a/searx/engines/tokyotoshokan.py
+++ b/searx/engines/tokyotoshokan.py
@@ -11,7 +11,6 @@
"""
import re
-from cgi import escape
from urllib import urlencode
from lxml import html
from searx.engines.xpath import extract_text
diff --git a/searx/engines/torrentz.py b/searx/engines/torrentz.py
index 92fbe7013..f9c832651 100644
--- a/searx/engines/torrentz.py
+++ b/searx/engines/torrentz.py
@@ -12,7 +12,6 @@
"""
import re
-from cgi import escape
from urllib import urlencode
from lxml import html
from searx.engines.xpath import extract_text
diff --git a/searx/engines/translated.py b/searx/engines/translated.py
index 02047bc93..e78db0d8e 100644
--- a/searx/engines/translated.py
+++ b/searx/engines/translated.py
@@ -9,7 +9,6 @@
@parse url, title, content
"""
import re
-from cgi import escape
from searx.utils import is_valid_lang
categories = ['general']
@@ -52,14 +51,14 @@ def request(query, params):
def response(resp):
results = []
results.append({
- 'url': escape(web_url.format(
+ 'url': web_url.format(
from_lang=resp.search_params['from_lang'][2],
to_lang=resp.search_params['to_lang'][2],
- query=resp.search_params['query'])),
- 'title': escape('[{0}-{1}] {2}'.format(
+ query=resp.search_params['query']),
+ 'title': '[{0}-{1}] {2}'.format(
resp.search_params['from_lang'][1],
resp.search_params['to_lang'][1],
- resp.search_params['query'])),
- 'content': escape(resp.json()['responseData']['translatedText'])
+ resp.search_params['query']),
+ 'content': resp.json()['responseData']['translatedText']
})
return results
diff --git a/searx/engines/wolframalpha_noapi.py b/searx/engines/wolframalpha_noapi.py
index e318d93e6..1534501b3 100644
--- a/searx/engines/wolframalpha_noapi.py
+++ b/searx/engines/wolframalpha_noapi.py
@@ -8,7 +8,6 @@
# @stable no
# @parse url, infobox
-from cgi import escape
from json import loads
from time import time
from urllib import urlencode
diff --git a/searx/engines/yandex.py b/searx/engines/yandex.py
index be3ec36ce..938fdd184 100644
--- a/searx/engines/yandex.py
+++ b/searx/engines/yandex.py
@@ -9,7 +9,6 @@
@parse url, title, content
"""
-from cgi import escape
from urllib import urlencode
from lxml import html
from searx.search import logger
@@ -52,8 +51,8 @@ def response(resp):
for result in dom.xpath(results_xpath):
try:
res = {'url': result.xpath(url_xpath)[0],
- 'title': escape(''.join(result.xpath(title_xpath))),
- 'content': escape(''.join(result.xpath(content_xpath)))}
+ 'title': ''.join(result.xpath(title_xpath)),
+ 'content': ''.join(result.xpath(content_xpath))}
except:
logger.exception('yandex parse crash')
continue
From 39f5035e1399713af7b11081e56d8f1a479a7b3a Mon Sep 17 00:00:00 2001
From: Adam Tauber <asciimoo@gmail.com>
Date: Fri, 9 Dec 2016 13:48:04 +0100
Subject: [PATCH 22/28] [fix] query escaping in rss/opensearch output
---
searx/templates/courgette/opensearch_response_rss.xml | 8 ++++----
searx/templates/legacy/opensearch_response_rss.xml | 8 ++++----
searx/templates/oscar/opensearch_response_rss.xml | 8 ++++----
3 files changed, 12 insertions(+), 12 deletions(-)
diff --git a/searx/templates/courgette/opensearch_response_rss.xml b/searx/templates/courgette/opensearch_response_rss.xml
index 5673eb2e1..ddb60fa5e 100644
--- a/searx/templates/courgette/opensearch_response_rss.xml
+++ b/searx/templates/courgette/opensearch_response_rss.xml
@@ -3,14 +3,14 @@
xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/"
xmlns:atom="http://www.w3.org/2005/Atom">
<channel>
- <title>Searx search: {{ q }}</title>
- <link>{{ base_url }}?q={{ q }}</link>
- <description>Search results for "{{ q }}" - searx</description>
+ <title>Searx search: {{ q|e }}</title>
+ <link>{{ base_url }}?q={{ q|e }}</link>
+ <description>Search results for "{{ q|e }}" - searx</description>
<opensearch:totalResults>{{ number_of_results }}</opensearch:totalResults>
<opensearch:startIndex>1</opensearch:startIndex>
<opensearch:itemsPerPage>{{ number_of_results }}</opensearch:itemsPerPage>
<atom:link rel="search" type="application/opensearchdescription+xml" href="{{ base_url }}opensearch.xml"/>
- <opensearch:Query role="request" searchTerms="{{ q }}" startPage="1" />
+ <opensearch:Query role="request" searchTerms="{{ q|e }}" startPage="1" />
{% for r in results %}
<item>
<title>{{ r.title }}</title>
diff --git a/searx/templates/legacy/opensearch_response_rss.xml b/searx/templates/legacy/opensearch_response_rss.xml
index 5673eb2e1..ddb60fa5e 100644
--- a/searx/templates/legacy/opensearch_response_rss.xml
+++ b/searx/templates/legacy/opensearch_response_rss.xml
@@ -3,14 +3,14 @@
xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/"
xmlns:atom="http://www.w3.org/2005/Atom">
<channel>
- <title>Searx search: {{ q }}</title>
- <link>{{ base_url }}?q={{ q }}</link>
- <description>Search results for "{{ q }}" - searx</description>
+ <title>Searx search: {{ q|e }}</title>
+ <link>{{ base_url }}?q={{ q|e }}</link>
+ <description>Search results for "{{ q|e }}" - searx</description>
<opensearch:totalResults>{{ number_of_results }}</opensearch:totalResults>
<opensearch:startIndex>1</opensearch:startIndex>
<opensearch:itemsPerPage>{{ number_of_results }}</opensearch:itemsPerPage>
<atom:link rel="search" type="application/opensearchdescription+xml" href="{{ base_url }}opensearch.xml"/>
- <opensearch:Query role="request" searchTerms="{{ q }}" startPage="1" />
+ <opensearch:Query role="request" searchTerms="{{ q|e }}" startPage="1" />
{% for r in results %}
<item>
<title>{{ r.title }}</title>
diff --git a/searx/templates/oscar/opensearch_response_rss.xml b/searx/templates/oscar/opensearch_response_rss.xml
index 5673eb2e1..ddb60fa5e 100644
--- a/searx/templates/oscar/opensearch_response_rss.xml
+++ b/searx/templates/oscar/opensearch_response_rss.xml
@@ -3,14 +3,14 @@
xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/"
xmlns:atom="http://www.w3.org/2005/Atom">
<channel>
- <title>Searx search: {{ q }}</title>
- <link>{{ base_url }}?q={{ q }}</link>
- <description>Search results for "{{ q }}" - searx</description>
+ <title>Searx search: {{ q|e }}</title>
+ <link>{{ base_url }}?q={{ q|e }}</link>
+ <description>Search results for "{{ q|e }}" - searx</description>
<opensearch:totalResults>{{ number_of_results }}</opensearch:totalResults>
<opensearch:startIndex>1</opensearch:startIndex>
<opensearch:itemsPerPage>{{ number_of_results }}</opensearch:itemsPerPage>
<atom:link rel="search" type="application/opensearchdescription+xml" href="{{ base_url }}opensearch.xml"/>
- <opensearch:Query role="request" searchTerms="{{ q }}" startPage="1" />
+ <opensearch:Query role="request" searchTerms="{{ q|e }}" startPage="1" />
{% for r in results %}
<item>
<title>{{ r.title }}</title>
From 7e1f27e45924147cc2219ddb9299460f202b206b Mon Sep 17 00:00:00 2001
From: Adam Tauber <asciimoo@gmail.com>
Date: Fri, 9 Dec 2016 18:48:54 +0100
Subject: [PATCH 23/28] [enh] add author to image result content
---
searx/templates/oscar/result_templates/images.html | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/searx/templates/oscar/result_templates/images.html b/searx/templates/oscar/result_templates/images.html
index f7bcf0786..b23f34915 100644
--- a/searx/templates/oscar/result_templates/images.html
+++ b/searx/templates/oscar/result_templates/images.html
@@ -13,7 +13,12 @@
</div>
<div class="modal-body">
<img class="img-responsive center-block" src="{% if result.thumbnail_src %}{{ image_proxify(result.thumbnail_src) }}{% else %}{{ image_proxify(result.img_src) }}{% endif %}" alt="{{ result.title|striptags }}">
- {% if result.content %}<p class="result-content">{{ result.content|safe }}</p>{% endif %}
+ {% if result.author %}<span class="photo-author">{{ result.author }}</span><br />{% endif %}
+ {% if result.content %}
+ <p class="result-content">
+ {{ result.content }}
+ </p>
+ {% endif %}
</div>
<div class="modal-footer">
<div class="clearfix"></div>
From ef2ef7974ab9bf55c5193a30544f71d8ca04f7b0 Mon Sep 17 00:00:00 2001
From: Adam Tauber <asciimoo@gmail.com>
Date: Fri, 9 Dec 2016 19:10:33 +0100
Subject: [PATCH 24/28] [enh] central html escaping of results
---
searx/webapp.py | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/searx/webapp.py b/searx/webapp.py
index 8996aa2b9..352a49fb6 100644
--- a/searx/webapp.py
+++ b/searx/webapp.py
@@ -40,7 +40,7 @@ except:
logger.critical("cannot import dependency: pygments")
from sys import exit
exit(1)
-
+from cgi import escape
from datetime import datetime, timedelta
from urllib import urlencode
from urlparse import urlparse, urljoin
@@ -433,8 +433,9 @@ def index():
for result in results:
if output_format == 'html':
if 'content' in result and result['content']:
- result['content'] = highlight_content(result['content'][:1024], search_query.query.encode('utf-8'))
- result['title'] = highlight_content(result['title'], search_query.query.encode('utf-8'))
+ result['content'] = highlight_content(escape(result['content'][:1024]),
+ search_query.query.encode('utf-8'))
+ result['title'] = highlight_content(escape(result['title']), search_query.query.encode('utf-8'))
else:
if result.get('content'):
result['content'] = html_to_text(result['content']).strip()
From fdf63940e8e576aee9fe5e03337d39e771f24514 Mon Sep 17 00:00:00 2001
From: Adam Tauber <asciimoo@gmail.com>
Date: Fri, 9 Dec 2016 19:10:57 +0100
Subject: [PATCH 25/28] [fix] tests ++ flickr error
---
searx/engines/flickr.py | 2 +-
tests/unit/engines/test_deezer.py | 2 +-
tests/unit/engines/test_flickr.py | 6 +++---
tests/unit/engines/test_flickr_noapi.py | 6 +++---
tests/unit/engines/test_kickass.py | 2 +-
tests/unit/engines/test_searchcode_doc.py | 3 ---
tests/unit/engines/test_spotify.py | 2 +-
7 files changed, 10 insertions(+), 13 deletions(-)
diff --git a/searx/engines/flickr.py b/searx/engines/flickr.py
index 1c3eef789..5ce1160e9 100644
--- a/searx/engines/flickr.py
+++ b/searx/engines/flickr.py
@@ -82,7 +82,7 @@ def response(resp):
'title': photo['title'],
'img_src': img_src,
'thumbnail_src': thumbnail_src,
- 'content': content = photo['description']['_content'],
+ 'content': photo['description']['_content'],
'author': photo['ownername'],
'template': 'images.html'})
diff --git a/tests/unit/engines/test_deezer.py b/tests/unit/engines/test_deezer.py
index cfef852af..5b9f55c33 100644
--- a/tests/unit/engines/test_deezer.py
+++ b/tests/unit/engines/test_deezer.py
@@ -42,7 +42,7 @@ class TestDeezerEngine(SearxTestCase):
self.assertEqual(len(results), 1)
self.assertEqual(results[0]['title'], 'Title of track')
self.assertEqual(results[0]['url'], 'https://www.deezer.com/track/1094042')
- self.assertEqual(results[0]['content'], 'Artist Name • Album Title • Title of track')
+ self.assertEqual(results[0]['content'], 'Artist Name - Album Title - Title of track')
self.assertTrue('100' in results[0]['embedded'])
json = r"""
diff --git a/tests/unit/engines/test_flickr.py b/tests/unit/engines/test_flickr.py
index 2d7472a92..be97647ce 100644
--- a/tests/unit/engines/test_flickr.py
+++ b/tests/unit/engines/test_flickr.py
@@ -52,7 +52,7 @@ class TestFlickrEngine(SearxTestCase):
self.assertEqual(results[0]['url'], 'https://www.flickr.com/photos/66847915@N08/15751017054')
self.assertTrue('o.jpg' in results[0]['img_src'])
self.assertTrue('n.jpg' in results[0]['thumbnail_src'])
- self.assertTrue('Owner' in results[0]['content'])
+ self.assertTrue('Owner' in results[0]['author'])
self.assertTrue('Description' in results[0]['content'])
json = r"""
@@ -76,7 +76,7 @@ class TestFlickrEngine(SearxTestCase):
self.assertEqual(results[0]['url'], 'https://www.flickr.com/photos/66847915@N08/15751017054')
self.assertTrue('z.jpg' in results[0]['img_src'])
self.assertTrue('z.jpg' in results[0]['thumbnail_src'])
- self.assertTrue('Owner' in results[0]['content'])
+ self.assertTrue('Owner' in results[0]['author'])
self.assertTrue('Description' in results[0]['content'])
json = r"""
@@ -100,7 +100,7 @@ class TestFlickrEngine(SearxTestCase):
self.assertEqual(results[0]['url'], 'https://www.flickr.com/photos/66847915@N08/15751017054')
self.assertTrue('o.jpg' in results[0]['img_src'])
self.assertTrue('o.jpg' in results[0]['thumbnail_src'])
- self.assertTrue('Owner' in results[0]['content'])
+ self.assertTrue('Owner' in results[0]['author'])
self.assertTrue('Description' in results[0]['content'])
json = r"""
diff --git a/tests/unit/engines/test_flickr_noapi.py b/tests/unit/engines/test_flickr_noapi.py
index 6d09071bd..5f8b069e3 100644
--- a/tests/unit/engines/test_flickr_noapi.py
+++ b/tests/unit/engines/test_flickr_noapi.py
@@ -145,7 +145,7 @@ class TestFlickrNoapiEngine(SearxTestCase):
self.assertEqual(results[0]['url'], 'https://www.flickr.com/photos/59729010@N00/14001294434')
self.assertIn('k.jpg', results[0]['img_src'])
self.assertIn('n.jpg', results[0]['thumbnail_src'])
- self.assertIn('Owner', results[0]['content'])
+ self.assertIn('Owner', results[0]['author'])
# no n size, only the z size
json = """
@@ -188,7 +188,7 @@ class TestFlickrNoapiEngine(SearxTestCase):
self.assertEqual(results[0]['url'], 'https://www.flickr.com/photos/59729010@N00/14001294434')
self.assertIn('z.jpg', results[0]['img_src'])
self.assertIn('z.jpg', results[0]['thumbnail_src'])
- self.assertIn('Owner', results[0]['content'])
+ self.assertIn('Owner', results[0]['author'])
# no z or n size
json = """
@@ -231,7 +231,7 @@ class TestFlickrNoapiEngine(SearxTestCase):
self.assertEqual(results[0]['url'], 'https://www.flickr.com/photos/59729010@N00/14001294434')
self.assertIn('o.jpg', results[0]['img_src'])
self.assertIn('o.jpg', results[0]['thumbnail_src'])
- self.assertIn('Owner', results[0]['content'])
+ self.assertIn('Owner', results[0]['author'])
# no image test
json = """
diff --git a/tests/unit/engines/test_kickass.py b/tests/unit/engines/test_kickass.py
index 96c17911c..7e1225da3 100644
--- a/tests/unit/engines/test_kickass.py
+++ b/tests/unit/engines/test_kickass.py
@@ -98,7 +98,7 @@ class TestKickassEngine(SearxTestCase):
self.assertEqual(len(results), 1)
self.assertEqual(results[0]['title'], 'This should be the title')
self.assertEqual(results[0]['url'], 'https://kickass.cd/url.html')
- self.assertEqual(results[0]['content'], 'Posted by riri in Other > Unsorted')
+ self.assertEqual(results[0]['content'], 'Posted by riri in Other > Unsorted')
self.assertEqual(results[0]['seed'], 10)
self.assertEqual(results[0]['leech'], 1)
self.assertEqual(results[0]['filesize'], 449)
diff --git a/tests/unit/engines/test_searchcode_doc.py b/tests/unit/engines/test_searchcode_doc.py
index 7228613ed..d02bb7a44 100644
--- a/tests/unit/engines/test_searchcode_doc.py
+++ b/tests/unit/engines/test_searchcode_doc.py
@@ -56,9 +56,6 @@ class TestSearchcodeDocEngine(SearxTestCase):
self.assertEqual(len(results), 1)
self.assertEqual(results[0]['title'], '[Type] Namespace test')
self.assertEqual(results[0]['url'], 'http://url')
- self.assertIn('Synopsis', results[0]['content'])
- self.assertIn('Type', results[0]['content'])
- self.assertIn('test', results[0]['content'])
self.assertIn('Description', results[0]['content'])
json = r"""
diff --git a/tests/unit/engines/test_spotify.py b/tests/unit/engines/test_spotify.py
index fd274abbd..e37c344d2 100644
--- a/tests/unit/engines/test_spotify.py
+++ b/tests/unit/engines/test_spotify.py
@@ -90,7 +90,7 @@ class TestSpotifyEngine(SearxTestCase):
self.assertEqual(len(results), 1)
self.assertEqual(results[0]['title'], 'Title of track')
self.assertEqual(results[0]['url'], 'https://open.spotify.com/track/2GzvFiedqW8hgqUpWcASZa')
- self.assertEqual(results[0]['content'], 'Artist Name • Album Title • Title of track')
+ self.assertEqual(results[0]['content'], 'Artist Name - Album Title - Title of track')
self.assertIn('1000', results[0]['embedded'])
json = """
From 8116e341cc26593a9527bf7bfbb140a80493fe31 Mon Sep 17 00:00:00 2001
From: Adam Tauber <asciimoo@gmail.com>
Date: Fri, 9 Dec 2016 19:20:55 +0100
Subject: [PATCH 26/28] [fix] kickass test html escaping
---
tests/unit/engines/test_kickass.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/tests/unit/engines/test_kickass.py b/tests/unit/engines/test_kickass.py
index 7e1225da3..3a75c6697 100644
--- a/tests/unit/engines/test_kickass.py
+++ b/tests/unit/engines/test_kickass.py
@@ -381,7 +381,7 @@ class TestKickassEngine(SearxTestCase):
self.assertEqual(len(results), 5)
self.assertEqual(results[0]['title'], 'This should be the title')
self.assertEqual(results[0]['url'], 'https://kickass.cd/url.html')
- self.assertEqual(results[0]['content'], 'Posted by riri in Other > Unsorted')
+ self.assertEqual(results[0]['content'], 'Posted by riri in Other > Unsorted')
self.assertEqual(results[0]['seed'], 10)
self.assertEqual(results[0]['leech'], 1)
self.assertEqual(results[0]['files'], 4)
From 72a217f983c935712eb872042157b4432eaec641 Mon Sep 17 00:00:00 2001
From: Adam Tauber <asciimoo@gmail.com>
Date: Fri, 9 Dec 2016 19:52:49 +0100
Subject: [PATCH 27/28] [mod] return empty string on missing osm title
Sometimes openstreetmap returns with None as title. In these cases use an empty
string instead.
---
searx/engines/openstreetmap.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/searx/engines/openstreetmap.py b/searx/engines/openstreetmap.py
index 38baaada9..01ca7d42d 100644
--- a/searx/engines/openstreetmap.py
+++ b/searx/engines/openstreetmap.py
@@ -43,7 +43,7 @@ def response(resp):
if 'display_name' not in r:
continue
- title = r['display_name']
+ title = r['display_name'] or u''
osm_type = r.get('osm_type', r.get('type'))
url = result_base_url.format(osm_type=osm_type,
osm_id=r['osm_id'])
From d80fb2c8e8995facb3a25c152c47a93eecf1fee4 Mon Sep 17 00:00:00 2001
From: Adam Tauber <asciimoo@gmail.com>
Date: Fri, 9 Dec 2016 19:57:28 +0100
Subject: [PATCH 28/28] [enh] central handling of empty result titles
---
searx/webapp.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/searx/webapp.py b/searx/webapp.py
index 352a49fb6..cdac52a45 100644
--- a/searx/webapp.py
+++ b/searx/webapp.py
@@ -435,7 +435,8 @@ def index():
if 'content' in result and result['content']:
result['content'] = highlight_content(escape(result['content'][:1024]),
search_query.query.encode('utf-8'))
- result['title'] = highlight_content(escape(result['title']), search_query.query.encode('utf-8'))
+ result['title'] = highlight_content(escape(result['title'] or u''),
+ search_query.query.encode('utf-8'))
else:
if result.get('content'):
result['content'] = html_to_text(result['content']).strip()