Merge branch 'plugins'

searx/plugins/__init__.py

@@ -0,0 +1,48 @@
+from searx.plugins import self_ip
+from searx import logger
+from sys import exit
+
+logger = logger.getChild('plugins')
+
+required_attrs = (('name', str),
+                  ('description', str),
+                  ('default_on', bool))
+
+
+class Plugin():
+    default_on = False
+    name = 'Default plugin'
+    description = 'Default plugin description'
+
+
+class PluginStore():
+
+    def __init__(self):
+        self.plugins = []
+
+    def __iter__(self):
+        for plugin in self.plugins:
+            yield plugin
+
+    def register(self, *plugins):
+        for plugin in plugins:
+            for plugin_attr, plugin_attr_type in required_attrs:
+                if not hasattr(plugin, plugin_attr) or not isinstance(getattr(plugin, plugin_attr), plugin_attr_type):
+                    logger.critical('missing attribute "{0}", cannot load plugin: {1}'.format(plugin_attr, plugin))
+                    exit(3)
+            plugin.id = plugin.name.replace(' ', '_')
+            self.plugins.append(plugin)
+
+    def call(self, plugin_type, request, *args, **kwargs):
+        ret = True
+        for plugin in request.user_plugins:
+            if hasattr(plugin, plugin_type):
+                ret = getattr(plugin, plugin_type)(request, *args, **kwargs)
+                if not ret:
+                    break
+
+        return ret
+
+
+plugins = PluginStore()
+plugins.register(self_ip)

searx/plugins/self_ip.py

@@ -0,0 +1,21 @@
+from flask.ext.babel import gettext
+name = "Self IP"
+description = gettext('Display your source IP address if the query expression is "ip"')
+default_on = True
+
+
+# attach callback to the pre search hook
+#  request: flask request object
+#  ctx: the whole local context of the pre search hook
+def pre_search(request, ctx):
+    if ctx['search'].query == 'ip':
+        x_forwarded_for = request.headers.getlist("X-Forwarded-For")
+        if x_forwarded_for:
+            ip = x_forwarded_for[0]
+        else:
+            ip = request.remote_addr
+        ctx['search'].answers.clear()
+        ctx['search'].answers.add(ip)
+        # return False prevents exeecution of the original block
+        return False
+    return True

searx/search.py

@@ -329,8 +329,8 @@ class Search(object):
         self.blocked_engines = get_blocked_engines(engines, request.cookies)
 
         self.results = []
-        self.suggestions = []
+        self.suggestions = set()
-        self.answers = []
+        self.answers = set()
         self.infoboxes = []
         self.request_data = {}
 
@@ -429,9 +429,6 @@ class Search(object):
         requests = []
         results_queue = Queue()
         results = {}
-        suggestions = set()
-        answers = set()
-        infoboxes = []
 
         # increase number of searches
         number_of_searches += 1
@@ -511,7 +508,7 @@ class Search(object):
                              selected_engine['name']))
 
         if not requests:
-            return results, suggestions, answers, infoboxes
+            return self
         # send all search-request
         threaded_requests(requests)
 
@@ -519,19 +516,19 @@ class Search(object):
             engine_name, engine_results = results_queue.get_nowait()
 
             # TODO type checks
-            [suggestions.add(x['suggestion'])
+            [self.suggestions.add(x['suggestion'])
              for x in list(engine_results)
              if 'suggestion' in x
              and engine_results.remove(x) is None]
 
-            [answers.add(x['answer'])
+            [self.answers.add(x['answer'])
              for x in list(engine_results)
              if 'answer' in x
              and engine_results.remove(x) is None]
 
-            infoboxes.extend(x for x in list(engine_results)
+            self.infoboxes.extend(x for x in list(engine_results)
-                             if 'infobox' in x
+                                  if 'infobox' in x
-                             and engine_results.remove(x) is None)
+                                  and engine_results.remove(x) is None)
 
             results[engine_name] = engine_results
 
@@ -541,16 +538,16 @@ class Search(object):
             engines[engine_name].stats['result_count'] += len(engine_results)
 
         # score results and remove duplications
-        results = score_results(results)
+        self.results = score_results(results)
 
         # merge infoboxes according to their ids
-        infoboxes = merge_infoboxes(infoboxes)
+        self.infoboxes = merge_infoboxes(self.infoboxes)
 
         # update engine stats, using calculated score
-        for result in results:
+        for result in self.results:
             for res_engine in result['engines']:
                 engines[result['engine']]\
                     .stats['score_count'] += result['score']
 
         # return results, suggestions, answers and infoboxes
-        return results, suggestions, answers, infoboxes
+        return self

searx/settings.yml

@@ -106,6 +106,7 @@ engines:
   - name : gigablast
     engine : gigablast
     shortcut : gb
+    disabled: True
 
   - name : github
     engine : github

searx/templates/oscar/macros.html

@@ -59,3 +59,11 @@
     </div>
     {% endif %}
 {%- endmacro %}
+
+{% macro checkbox_toggle(id, blocked) -%}
+    <div class="checkbox">
+        <input class="hidden" type="checkbox" id="{{ id }}" name="{{ id }}"{% if blocked %} checked="checked"{% endif %} />
+        <label class="btn btn-success label_hide_if_checked" for="{{ id }}">{{ _('Block') }}</label>
+        <label class="btn btn-danger label_hide_if_not_checked" for="{{ id }}">{{ _('Allow') }}</label>
+    </div>
+{%- endmacro %}

searx/templates/oscar/preferences.html

@@ -1,4 +1,4 @@
-{% from 'oscar/macros.html' import preferences_item_header, preferences_item_header_rtl, preferences_item_footer, preferences_item_footer_rtl %}
+{% from 'oscar/macros.html' import preferences_item_header, preferences_item_header_rtl, preferences_item_footer, preferences_item_footer_rtl, checkbox_toggle %}
 {% extends "oscar/base.html" %}
 {% block title %}{{ _('preferences') }} - {% endblock %}
 {% block site_alert_warning_nojs %}
@@ -16,6 +16,7 @@
     <ul class="nav nav-tabs nav-justified hide_if_nojs" role="tablist" style="margin-bottom:20px;">
       <li class="active"><a href="#tab_general" role="tab" data-toggle="tab">{{ _('General') }}</a></li>
       <li><a href="#tab_engine" role="tab" data-toggle="tab">{{ _('Engines') }}</a></li>
+      <li><a href="#tab_plugins" role="tab" data-toggle="tab">{{ _('Plugins') }}</a></li>
     </ul>
 
     <!-- Tab panes -->
@@ -139,11 +140,7 @@
                                 <div class="col-xs-6 col-sm-4 col-md-4">{{ search_engine.name }} ({{ shortcuts[search_engine.name] }})</div>
                                 {% endif %}
                                 <div class="col-xs-6 col-sm-4 col-md-4">
-                                    <div class="checkbox">
+                                    {{ checkbox_toggle('engine_' + search_engine.name|replace(' ', '_') + '__' + categ|replace(' ', '_'), (search_engine.name, categ) in blocked_engines) }}
-                                    <input class="hidden" type="checkbox" id="engine_{{ categ|replace(' ', '_') }}_{{ search_engine.name|replace(' ', '_') }}" name="engine_{{ search_engine.name }}__{{ categ }}"{% if (search_engine.name, categ) in blocked_engines %} checked="checked"{% endif %} />
-                                    <label class="btn btn-success label_hide_if_checked" for="engine_{{ categ|replace(' ', '_') }}_{{ search_engine.name|replace(' ', '_') }}">{{ _('Block') }}</label>
-                                    <label class="btn btn-danger label_hide_if_not_checked" for="engine_{{ categ|replace(' ', '_') }}_{{ search_engine.name|replace(' ', '_') }}">{{ _('Allow') }}</label>
-                                    </div>
                                 </div>
                                 {% if rtl %}
                                 <div class="col-xs-6 col-sm-4 col-md-4">{{ search_engine.name }} ({{ shortcuts[search_engine.name] }})&lrm;</div>
@@ -157,6 +154,28 @@
                 {% endfor %}
             </div>
         </div>
+        <div class="tab-pane active_if_nojs" id="tab_plugins">
+            <noscript>
+                <h3>{{ _('Plugins') }}</h3>
+            </noscript>
+            <fieldset>
+            <div class="container-fluid">
+                {% for plugin in plugins %}
+                <div class="panel panel-default">
+                    <div class="panel-heading">
+                        <h3 class="panel-title">{{ plugin.name }}</h3>
+                    </div>
+                    <div class="panel-body">
+                        <div class="col-xs-6 col-sm-4 col-md-6">{{ plugin.description }}</div>
+                        <div class="col-xs-6 col-sm-4 col-md-6">
+                            {{ checkbox_toggle('plugin_' + plugin.id, plugin.id not in allowed_plugins) }}
+                        </div>
+                    </div>
+                </div>
+                {% endfor %}
+            </div>
+            </fieldset>
+        </div>
     </div>
     <p class="text-muted" style="margin:20px 0;">{{ _('These settings are stored in your cookies, this allows us not to store this data about you.') }}
     <br />

searx/templates/oscar/results.html

@@ -26,7 +26,7 @@
             </div>
             {% endfor %}
 
-            {% if not results %}
+            {% if not results and not answers %}
                 {% include 'oscar/messages/no_results.html' %}
             {% endif %}
 

searx/tests/test_plugins.py

@@ -0,0 +1,51 @@
+# -*- coding: utf-8 -*-
+
+from searx.testing import SearxTestCase
+from searx import plugins
+from mock import Mock
+
+
+class PluginStoreTest(SearxTestCase):
+
+    def test_PluginStore_init(self):
+        store = plugins.PluginStore()
+        self.assertTrue(isinstance(store.plugins, list) and len(store.plugins) == 0)
+
+    def test_PluginStore_register(self):
+        store = plugins.PluginStore()
+        testplugin = plugins.Plugin()
+        store.register(testplugin)
+
+        self.assertTrue(len(store.plugins) == 1)
+
+    def test_PluginStore_call(self):
+        store = plugins.PluginStore()
+        testplugin = plugins.Plugin()
+        store.register(testplugin)
+        setattr(testplugin, 'asdf', Mock())
+        request = Mock(user_plugins=[])
+        store.call('asdf', request, Mock())
+
+        self.assertFalse(testplugin.asdf.called)
+
+        request.user_plugins.append(testplugin)
+        store.call('asdf', request, Mock())
+
+        self.assertTrue(testplugin.asdf.called)
+
+
+class SelfIPTest(SearxTestCase):
+
+    def test_PluginStore_init(self):
+        store = plugins.PluginStore()
+        store.register(plugins.self_ip)
+
+        self.assertTrue(len(store.plugins) == 1)
+
+        request = Mock(user_plugins=store.plugins,
+                       remote_addr='127.0.0.1')
+        request.headers.getlist.return_value = []
+        ctx = {'search': Mock(answers=set(),
+                              query='ip')}
+        store.call('pre_search', request, ctx)
+        self.assertTrue('127.0.0.1' in ctx['search'].answers)

searx/tests/test_webapp.py

@@ -2,7 +2,6 @@
 
 import json
 from urlparse import ParseResult
-from mock import patch
 from searx import webapp
 from searx.testing import SearxTestCase
 
@@ -33,6 +32,11 @@ class ViewsTestCase(SearxTestCase):
             },
         ]
 
+        def search_mock(search_self, *args):
+            search_self.results = self.test_results
+
+        webapp.Search.search = search_mock
+
         self.maxDiff = None  # to see full diffs
 
     def test_index_empty(self):
@@ -40,14 +44,7 @@ class ViewsTestCase(SearxTestCase):
         self.assertEqual(result.status_code, 200)
         self.assertIn('<div class="title"><h1>searx</h1></div>', result.data)
 
-    @patch('searx.search.Search.search')
+    def test_index_html(self):
-    def test_index_html(self, search):
-        search.return_value = (
-            self.test_results,
-            set(),
-            set(),
-            set()
-        )
         result = self.app.post('/', data={'q': 'test'})
         self.assertIn(
             '<h3 class="result_title"><img width="14" height="14" class="favicon" src="/static/themes/default/img/icons/icon_youtube.ico" alt="youtube" /><a href="http://second.test.xyz">Second <span class="highlight">Test</span></a></h3>',  # noqa
@@ -58,14 +55,7 @@ class ViewsTestCase(SearxTestCase):
             result.data
         )
 
-    @patch('searx.search.Search.search')
+    def test_index_json(self):
-    def test_index_json(self, search):
-        search.return_value = (
-            self.test_results,
-            set(),
-            set(),
-            set()
-        )
         result = self.app.post('/', data={'q': 'test', 'format': 'json'})
 
         result_dict = json.loads(result.data)
@@ -76,14 +66,7 @@ class ViewsTestCase(SearxTestCase):
         self.assertEqual(
             result_dict['results'][0]['url'], 'http://first.test.xyz')
 
-    @patch('searx.search.Search.search')
+    def test_index_csv(self):
-    def test_index_csv(self, search):
-        search.return_value = (
-            self.test_results,
-            set(),
-            set(),
-            set()
-        )
         result = self.app.post('/', data={'q': 'test', 'format': 'csv'})
 
         self.assertEqual(
@@ -93,14 +76,7 @@ class ViewsTestCase(SearxTestCase):
             result.data
         )
 
-    @patch('searx.search.Search.search')
+    def test_index_rss(self):
-    def test_index_rss(self, search):
-        search.return_value = (
-            self.test_results,
-            set(),
-            set(),
-            set()
-        )
         result = self.app.post('/', data={'q': 'test', 'format': 'rss'})
 
         self.assertIn(

searx/webapp.py

@@ -27,6 +27,18 @@ import cStringIO
 import os
 import hashlib
 
+from searx import logger
+logger = logger.getChild('webapp')
+
+try:
+    from pygments import highlight
+    from pygments.lexers import get_lexer_by_name
+    from pygments.formatters import HtmlFormatter
+except:
+    logger.critical("cannot import dependency: pygments")
+    from sys import exit
+    exit(1)
+
 from datetime import datetime, timedelta
 from urllib import urlencode
 from werkzeug.contrib.fixers import ProxyFix
@@ -51,19 +63,9 @@ from searx.https_rewrite import https_url_rewrite
 from searx.search import Search
 from searx.query import Query
 from searx.autocomplete import searx_bang, backends as autocomplete_backends
-from searx import logger
+from searx.plugins import plugins
-try:
-    from pygments import highlight
-    from pygments.lexers import get_lexer_by_name
-    from pygments.formatters import HtmlFormatter
-except:
-    logger.critical("cannot import dependency: pygments")
-    from sys import exit
-    exit(1)
 
 
-logger = logger.getChild('webapp')
-
 static_path, templates_path, themes =\
     get_themes(settings['themes_path']
                if settings.get('themes_path')
@@ -303,6 +305,23 @@ def render(template_name, override_theme=None, **kwargs):
         '{}/{}'.format(kwargs['theme'], template_name), **kwargs)
 
 
+@app.before_request
+def pre_request():
+    # merge GET, POST vars
+    request.form = dict(request.form.items())
+    for k, v in request.args:
+        if k not in request.form:
+            request.form[k] = v
+
+    request.user_plugins = []
+    allowed_plugins = request.cookies.get('allowed_plugins', '').split(',')
+    disabled_plugins = request.cookies.get('disabled_plugins', '').split(',')
+    for plugin in plugins:
+        if ((plugin.default_on and plugin.id not in disabled_plugins)
+                or plugin.id in allowed_plugins):
+            request.user_plugins.append(plugin)
+
+
 @app.route('/search', methods=['GET', 'POST'])
 @app.route('/', methods=['GET', 'POST'])
 def index():
@@ -323,8 +342,10 @@ def index():
             'index.html',
         )
 
-    search.results, search.suggestions,\
+    if plugins.call('pre_search', request, locals()):
-        search.answers, search.infoboxes = search.search(request)
+        search.search(request)
+
+    plugins.call('post_search', request, locals())
 
     for result in search.results:
 
@@ -487,11 +508,11 @@ def preferences():
         blocked_engines = get_blocked_engines(engines, request.cookies)
     else:  # on save
         selected_categories = []
+        post_disabled_plugins = []
         locale = None
         autocomplete = ''
         method = 'POST'
         safesearch = '1'
-
         for pd_name, pd in request.form.items():
             if pd_name.startswith('category_'):
                 category = pd_name[9:]
@@ -514,14 +535,34 @@ def preferences():
                 safesearch = pd
             elif pd_name.startswith('engine_'):
                 if pd_name.find('__') > -1:
-                    engine_name, category = pd_name.replace('engine_', '', 1).split('__', 1)
+                    # TODO fix underscore vs space
+                    engine_name, category = [x.replace('_', ' ') for x in
+                                             pd_name.replace('engine_', '', 1).split('__', 1)]
                     if engine_name in engines and category in engines[engine_name].categories:
                         blocked_engines.append((engine_name, category))
             elif pd_name == 'theme':
                 theme = pd if pd in themes else default_theme
+            elif pd_name.startswith('plugin_'):
+                plugin_id = pd_name.replace('plugin_', '', 1)
+                if not any(plugin.id == plugin_id for plugin in plugins):
+                    continue
+                post_disabled_plugins.append(plugin_id)
             else:
                 resp.set_cookie(pd_name, pd, max_age=cookie_max_age)
 
+        disabled_plugins = []
+        allowed_plugins = []
+        for plugin in plugins:
+            if plugin.default_on:
+                if plugin.id in post_disabled_plugins:
+                    disabled_plugins.append(plugin.id)
+            elif plugin.id not in post_disabled_plugins:
+                allowed_plugins.append(plugin.id)
+
+        resp.set_cookie('disabled_plugins', ','.join(disabled_plugins), max_age=cookie_max_age)
+
+        resp.set_cookie('allowed_plugins', ','.join(allowed_plugins), max_age=cookie_max_age)
+
         resp.set_cookie(
             'blocked_engines', ','.join('__'.join(e) for e in blocked_engines),
             max_age=cookie_max_age
@@ -571,6 +612,8 @@ def preferences():
                   autocomplete_backends=autocomplete_backends,
                   shortcuts={y: x for x, y in engine_shortcuts.items()},
                   themes=themes,
+                  plugins=plugins,
+                  allowed_plugins=[plugin.id for plugin in request.user_plugins],
                   theme=get_current_theme_name())