Merge branch 'master' of https://github.com/asciimoo/searx

Conflicts:
	searx/tests/test_engines.py

.gitignore

@@ -5,24 +5,13 @@ env
 robot_log.html
 robot_output.xml
 robot_report.html
+test_basic/
 setup.cfg
 
 *.pyc
 */*.pyc
 *~
 
-bin/
-build/
-coverage/
-develop-eggs/
-dist/
-eggs/
-include/
-lib/
-local/
-parts/
-searx.egg-info/
-var/
 node_modules/
 
 .tx/

.travis.yml

@@ -11,16 +11,16 @@ before_install:
   - "export DISPLAY=:99.0"
   - "sh -e /etc/init.d/xvfb start"
   - npm install -g less grunt-cli
-  - ( cd searx/static/themes/oscar;npm install )
+  - ( cd searx/static/themes/oscar;npm install; cd - )
 install:
-  - "make"
+  - ./manage.sh update_dev_packages
   - pip install coveralls
 script:
-  - "make flake8"
-  - "make robot"
-  - "make styles"
-  - "make grunt"
-  - make coverage
+  - ./manage.sh pep8_check
+  - ./manage.sh styles
+  - ./manage.sh grunt_build
+  - ./manage.sh py_test_coverage
+  - ./manage.sh robot_tests
 after_success:
   coveralls
 notifications:

Makefile

@@ -1,66 +0,0 @@
-# convenience makefile to boostrap & run buildout
-# use `make options=-v` to run buildout with extra options
-
-version = 2.7
-python = bin/python
-options =
-
-all: .installed.cfg
-
-.installed.cfg: bin/buildout buildout.cfg setup.py
-	bin/buildout $(options)
-
-bin/buildout: $(python) buildout.cfg bootstrap.py
-	$(python) bootstrap.py
-	@touch $@
-
-$(python):
-	virtualenv -p python$(version) --no-site-packages .
-	@touch $@
-
-robot: .installed.cfg
-	@bin/robot
-
-flake8: .installed.cfg
-	@bin/flake8 setup.py
-	@bin/flake8 ./searx/
-
-tests: .installed.cfg flake8
-	@bin/test
-	@grunt test --gruntfile searx/static/themes/oscar/gruntfile.js
-
-coverage: .installed.cfg
-	@bin/coverage run bin/test
-	@bin/coverage report
-	@bin/coverage html
-
-production: bin/buildout production.cfg setup.py
-	bin/buildout -c production.cfg $(options)
-	@echo "* Please modify `readlink --canonicalize-missing ./searx/settings.py`"
-	@echo "* Hint 1: on production, disable debug mode and change secret_key"
-	@echo "* Hint 2: searx will be executed at server startup by crontab"
-	@echo "* Hint 3: to run immediatley, execute 'bin/supervisord'"
-
-minimal: bin/buildout minimal.cfg setup.py
-	bin/buildout -c minimal.cfg $(options)
-
-styles:
-	@lessc -x searx/static/themes/default/less/style.less > searx/static/themes/default/css/style.css
-	@lessc -x searx/static/themes/default/less/style-rtl.less > searx/static/themes/default/css/style-rtl.css
-	@lessc -x searx/static/themes/courgette/less/style.less > searx/static/themes/courgette/css/style.css
-	@lessc -x searx/static/themes/courgette/less/style-rtl.less > searx/static/themes/courgette/css/style-rtl.css
-	@lessc -x searx/static/less/bootstrap/bootstrap.less > searx/static/css/bootstrap.min.css
-	@lessc -x searx/static/themes/oscar/less/oscar/oscar.less > searx/static/themes/oscar/css/oscar.min.css
-	@lessc -x searx/static/themes/pix-art/less/style.less > searx/static/themes/pix-art/css/style.css
-
-grunt:
-	@grunt --gruntfile searx/static/themes/oscar/gruntfile.js
-
-locales:
-	@pybabel compile -d searx/translations
-
-clean:
-	@rm -rf .installed.cfg .mr.developer.cfg bin parts develop-eggs eggs \
-		searx.egg-info lib include .coverage coverage
-
-.PHONY: all tests robot flake8 coverage production minimal styles locales clean

README.rst

@@ -7,31 +7,16 @@ engine <https://en.wikipedia.org/wiki/Metasearch_engine>`__.
 List of `running
 instances <https://github.com/asciimoo/searx/wiki/Searx-instances>`__.
 
-See the `wiki <https://github.com/asciimoo/searx/wiki>`__ for more information.
+See the `documentation <https://asciimoo.github.io/searx>`__ and the `wiki <https://github.com/asciimoo/searx/wiki>`__ for more information.
 
 |Flattr searx|
 
-Features
-~~~~~~~~
-
-- Tracking free
-- Supports multiple output formats
-
-  - json ``curl https://searx.me/?format=json&q=[query]``
-  - csv ``curl https://searx.me/?format=csv&q=[query]``
-  - opensearch/rss ``curl https://searx.me/?format=rss&q=[query]``
-- Opensearch support (you can set as default search engine)
-- Configurable search engines/categories
-- Different search languages
-- Duckduckgo like !bang functionality with engine shortcuts
-- Parallel queries - relatively fast
-
 Installation
 ~~~~~~~~~~~~
 
 -  clone source:
    ``git clone git@github.com:asciimoo/searx.git && cd searx``
--  install dependencies: ``pip install -r requirements.txt``
+-  install dependencies: ``./manage.sh update_packages``
 -  edit your
    `settings.yml <https://github.com/asciimoo/searx/blob/master/settings.yml>`__
    (set your ``secret_key``!)
@@ -40,104 +25,6 @@ Installation
 For all the details, follow this `step by step
 installation <https://github.com/asciimoo/searx/wiki/Installation>`__
 
-Alternative (Recommended) Installation
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
--  clone source:
-   ``git clone git@github.com:asciimoo/searx.git && cd searx``
--  build in current folder: ``make minimal``
--  run ``bin/searx-run`` to start the application
-
-Development
-~~~~~~~~~~~
-
-Just run ``make``. Versions of dependencies are pinned down inside
-``versions.cfg`` to produce most stable build. Also remember, NO make
-command should be run as root, not even ``make production``
-
-Deployment
-~~~~~~~~~~
-
--  clone source:
-   ``git clone git@github.com:asciimoo/searx.git && cd searx``
--  build in current folder: ``make production``
--  run ``bin/supervisord`` to start the application
-
-Upgrading
-~~~~~~~~~
-
--  inside previously cloned searx directory run: ``git stash`` to
-   temporarily save any changes you have made
--  pull source: ``git pull origin master``
--  re-build in current folder: ``make production``
--  run ``bin/supervisorctl stop searx`` to stop searx, if it does not,
-   then run ``fuser -k 8888/tcp``
--  run ``bin/supervisorctl reload`` to re-read supervisor config and
-   start searx
-
-Command make
-~~~~~~~~~~~~
-
-``make``
-''''''''
-
-Builds development environment with testing support.
-
-``make tests``
-''''''''''''''
-
-Runs tests. You can write tests
-`here <https://github.com/asciimoo/searx/tree/master/searx/tests>`__ and
-remember 'untested code is broken code'.
-
-``make robot``
-''''''''''''''
-
-Runs robot (Selenium) tests, you must have ``firefox`` installed because
-this functional tests actually run the browser and perform operations on
-it. Also searx is executed with
-`settings\_robot <https://github.com/asciimoo/searx/blob/master/searx/settings_robot.yml>`__.
-
-``make flake8``
-'''''''''''''''
-
-'pep8 is a tool to check your Python code against some of the style
-conventions in `PEP 8 <http://www.python.org/dev/peps/pep-0008/>`__.'
-
-``make coverage``
-'''''''''''''''''
-
-Checks coverage of tests, after running this, execute this:
-``firefox ./coverage/index.html``
-
-``make production``
-'''''''''''''''''''
-
-Used to make co-called production environment - without tests (you
-should ran tests before deploying searx on the server). This installs
-supervisord, so if searx crashes, it will try to pick itself up again.
-And crontab entry is added to start supervisord at server boot.
-
-``make minimal``
-''''''''''''''''
-
-Minimal build - without test frameworks, the quickest build option.
-
-``make clean``
-''''''''''''''
-
-Deletes several folders and files (see ``Makefile`` for more), so that
-next time you run any other ``make`` command it will rebuild everithing.
-
-TODO
-~~~~
-
-- Moar engines
-- Better ui
-- Browser integration
-- Documentation
-- Tests
-
 Bugs
 ~~~~
 

base.cfg

@@ -1,17 +0,0 @@
-[buildout]
-extends = versions.cfg
-unzip = true
-newest = false
-prefer-final = true
-develop = .
-
-eggs =
-    searx
-
-parts =
-    omelette
-
-
-[omelette]
-recipe = collective.recipe.omelette
-eggs = ${buildout:eggs}

bootstrap.py

@@ -1,210 +0,0 @@
-##############################################################################
-#
-# Copyright (c) 2006 Zope Foundation and Contributors.
-# All Rights Reserved.
-#
-# This software is subject to the provisions of the Zope Public License,
-# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
-# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
-# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
-# FOR A PARTICULAR PURPOSE.
-#
-##############################################################################
-"""Bootstrap a buildout-based project
-
-Simply run this script in a directory containing a buildout.cfg.
-The script accepts buildout command-line options, so you can
-use the -c option to specify an alternate configuration file.
-"""
-
-import os
-import shutil
-import sys
-import tempfile
-
-from optparse import OptionParser
-
-__version__ = '2015-07-01'
-# See zc.buildout's changelog if this version is up to date.
-
-tmpeggs = tempfile.mkdtemp(prefix='bootstrap-')
-
-usage = '''\
-[DESIRED PYTHON FOR BUILDOUT] bootstrap.py [options]
-
-Bootstraps a buildout-based project.
-
-Simply run this script in a directory containing a buildout.cfg, using the
-Python that you want bin/buildout to use.
-
-Note that by using --find-links to point to local resources, you can keep
-this script from going over the network.
-'''
-
-parser = OptionParser(usage=usage)
-parser.add_option("--version",
-                  action="store_true", default=False,
-                  help=("Return bootstrap.py version."))
-parser.add_option("-t", "--accept-buildout-test-releases",
-                  dest='accept_buildout_test_releases',
-                  action="store_true", default=False,
-                  help=("Normally, if you do not specify a --version, the "
-                        "bootstrap script and buildout gets the newest "
-                        "*final* versions of zc.buildout and its recipes and "
-                        "extensions for you.  If you use this flag, "
-                        "bootstrap and buildout will get the newest releases "
-                        "even if they are alphas or betas."))
-parser.add_option("-c", "--config-file",
-                  help=("Specify the path to the buildout configuration "
-                        "file to be used."))
-parser.add_option("-f", "--find-links",
-                  help=("Specify a URL to search for buildout releases"))
-parser.add_option("--allow-site-packages",
-                  action="store_true", default=False,
-                  help=("Let bootstrap.py use existing site packages"))
-parser.add_option("--buildout-version",
-                  help="Use a specific zc.buildout version")
-parser.add_option("--setuptools-version",
-                  help="Use a specific setuptools version")
-parser.add_option("--setuptools-to-dir",
-                  help=("Allow for re-use of existing directory of "
-                        "setuptools versions"))
-
-options, args = parser.parse_args()
-if options.version:
-    print("bootstrap.py version %s" % __version__)
-    sys.exit(0)
-
-
-######################################################################
-# load/install setuptools
-
-try:
-    from urllib.request import urlopen
-except ImportError:
-    from urllib2 import urlopen
-
-ez = {}
-if os.path.exists('ez_setup.py'):
-    exec(open('ez_setup.py').read(), ez)
-else:
-    exec(urlopen('https://bootstrap.pypa.io/ez_setup.py').read(), ez)
-
-if not options.allow_site_packages:
-    # ez_setup imports site, which adds site packages
-    # this will remove them from the path to ensure that incompatible versions
-    # of setuptools are not in the path
-    import site
-    # inside a virtualenv, there is no 'getsitepackages'.
-    # We can't remove these reliably
-    if hasattr(site, 'getsitepackages'):
-        for sitepackage_path in site.getsitepackages():
-            # Strip all site-packages directories from sys.path that
-            # are not sys.prefix; this is because on Windows
-            # sys.prefix is a site-package directory.
-            if sitepackage_path != sys.prefix:
-                sys.path[:] = [x for x in sys.path
-                               if sitepackage_path not in x]
-
-setup_args = dict(to_dir=tmpeggs, download_delay=0)
-
-if options.setuptools_version is not None:
-    setup_args['version'] = options.setuptools_version
-if options.setuptools_to_dir is not None:
-    setup_args['to_dir'] = options.setuptools_to_dir
-
-ez['use_setuptools'](**setup_args)
-import setuptools
-import pkg_resources
-
-# This does not (always?) update the default working set.  We will
-# do it.
-for path in sys.path:
-    if path not in pkg_resources.working_set.entries:
-        pkg_resources.working_set.add_entry(path)
-
-######################################################################
-# Install buildout
-
-ws = pkg_resources.working_set
-
-setuptools_path = ws.find(
-    pkg_resources.Requirement.parse('setuptools')).location
-
-# Fix sys.path here as easy_install.pth added before PYTHONPATH
-cmd = [sys.executable, '-c',
-       'import sys; sys.path[0:0] = [%r]; ' % setuptools_path +
-       'from setuptools.command.easy_install import main; main()',
-       '-mZqNxd', tmpeggs]
-
-find_links = os.environ.get(
-    'bootstrap-testing-find-links',
-    options.find_links or
-    ('http://downloads.buildout.org/'
-     if options.accept_buildout_test_releases else None)
-    )
-if find_links:
-    cmd.extend(['-f', find_links])
-
-requirement = 'zc.buildout'
-version = options.buildout_version
-if version is None and not options.accept_buildout_test_releases:
-    # Figure out the most recent final version of zc.buildout.
-    import setuptools.package_index
-    _final_parts = '*final-', '*final'
-
-    def _final_version(parsed_version):
-        try:
-            return not parsed_version.is_prerelease
-        except AttributeError:
-            # Older setuptools
-            for part in parsed_version:
-                if (part[:1] == '*') and (part not in _final_parts):
-                    return False
-            return True
-
-    index = setuptools.package_index.PackageIndex(
-        search_path=[setuptools_path])
-    if find_links:
-        index.add_find_links((find_links,))
-    req = pkg_resources.Requirement.parse(requirement)
-    if index.obtain(req) is not None:
-        best = []
-        bestv = None
-        for dist in index[req.project_name]:
-            distv = dist.parsed_version
-            if _final_version(distv):
-                if bestv is None or distv > bestv:
-                    best = [dist]
-                    bestv = distv
-                elif distv == bestv:
-                    best.append(dist)
-        if best:
-            best.sort()
-            version = best[-1].version
-if version:
-    requirement = '=='.join((requirement, version))
-cmd.append(requirement)
-
-import subprocess
-if subprocess.call(cmd) != 0:
-    raise Exception(
-        "Failed to execute command:\n%s" % repr(cmd)[1:-1])
-
-######################################################################
-# Import and run buildout
-
-ws.add_entry(tmpeggs)
-ws.require(requirement)
-import zc.buildout.buildout
-
-if not [a for a in args if '=' not in a]:
-    args.append('bootstrap')
-
-# if -c was provided, we push it back into args for buildout' main function
-if options.config_file is not None:
-    args[0:0] = ['-c', options.config_file]
-
-zc.buildout.buildout.main(args)
-shutil.rmtree(tmpeggs)

buildout.cfg

@@ -1,30 +0,0 @@
-[buildout]
-extends = base.cfg
-develop = .
-
-eggs =
-    searx [test]
-
-parts +=
-    pyscripts
-    robot
-    test
-
-
-[pyscripts]
-recipe = zc.recipe.egg:script
-eggs = ${buildout:eggs}
-interpreter = py
-dependent-scripts = true
-
-
-[robot]
-recipe = zc.recipe.testrunner
-eggs = ${buildout:eggs}
-defaults = ['--color', '--auto-progress', '--layer', 'SearxRobotLayer']
-
-
-[test]
-recipe = zc.recipe.testrunner
-eggs = ${buildout:eggs}
-defaults = ['--color', '--auto-progress', '--layer', 'SearxTestLayer', '--layer', '!SearxRobotLayer']

manage.sh

@@ -0,0 +1,94 @@
+#!/bin/sh
+
+BASE_DIR=$(dirname `readlink -f $0`)
+PYTHONPATH=$BASE_DIR
+SEARX_DIR="$BASE_DIR/searx"
+ACTION=$1
+
+update_packages() {
+    pip install --upgrade -r "$BASE_DIR/requirements.txt"
+}
+
+update_dev_packages() {
+    update_packages
+    pip install --upgrade -r "$BASE_DIR/requirements-dev.txt"
+}
+
+pep8_check() {
+    echo '[!] Running pep8 check'
+    pep8 --max-line-length=120 "$SEARX_DIR" "$BASE_DIR/tests"
+}
+
+unit_tests() {
+    echo '[!] Running unit tests'
+    python -m nose2 -s "$BASE_DIR/tests/unit"
+}
+
+py_test_coverage() {
+    echo '[!] Running python test coverage'
+    PYTHONPATH=`pwd` python -m nose2 -C --coverage "$SEARX_DIR" -s "$BASE_DIR/tests/unit"
+    coverage report
+    coverage html
+}
+
+robot_tests() {
+    echo '[!] Running robot tests'
+    PYTHONPATH=`pwd` python "$SEARX_DIR/testing.py" robot
+}
+
+tests() {
+    set -e
+    pep8_check
+    unit_tests
+    robot_tests
+    set +e
+}
+
+build_style() {
+    lessc -x "$BASE_DIR/searx/static/$1" "$BASE_DIR/searx/static/$2"
+}
+
+styles() {
+    echo '[!] Building styles'
+	build_style themes/default/less/style.less themes/default/css/style.css
+	build_style themes/default/less/style-rtl.less themes/default/css/style-rtl.css
+	build_style themes/courgette/less/style.less themes/courgette/css/style.css
+	build_style themes/courgette/less/style-rtl.less themes/courgette/css/style-rtl.css
+	build_style less/bootstrap/bootstrap.less css/bootstrap.min.css
+	build_style themes/oscar/less/oscar/oscar.less themes/oscar/css/oscar.min.css
+	build_style themes/pix-art/less/style.less themes/pix-art/css/style.css
+}
+
+grunt_build() {
+	grunt --gruntfile "$SEARX_DIR/static/themes/oscar/gruntfile.js"
+}
+
+locales() {
+	pybabel compile -d "$SEARX_DIR/translations"
+}
+
+help() {
+    [ -z "$1" ] || echo "Error: $1\n"
+    echo "Searx manage.sh help
+
+Commands
+========
+    grunt_build          - Build js files
+    help                 - This text
+    locales              - Compile locales
+    pep8_check           - Pep8 validation
+    py_test_coverage     - Unit test coverage
+    robot_tests          - Run selenium tests
+    styles               - Build less files
+    tests                - Run all python tests (pep8, unit, robot)
+    unit_tests           - Run unit tests
+    update_dev_packages  - Check & update development and production dependency changes
+    update_packages      - Check & update dependency changes
+"
+}
+
+if type $ACTION 1>/dev/null; then
+    $ACTION
+else
+    help "action not found"
+fi

minimal.cfg

@@ -1,15 +0,0 @@
-[buildout]
-extends = base.cfg
-develop = .
-
-eggs =
-    searx
-
-parts +=
-    pyscripts
-
-
-[pyscripts]
-recipe = zc.recipe.egg:script
-eggs = ${buildout:eggs}
-interpreter = py

production.cfg

@@ -1,34 +0,0 @@
-[buildout]
-extends = base.cfg
-develop = .
-
-eggs =
-    searx
-
-parts +=
-    pyscripts
-    supervisor
-    crontab_reboot
-
-
-[pyscripts]
-recipe = zc.recipe.egg:script
-eggs = ${buildout:eggs}
-interpreter = py
-
-
-[supervisor]
-recipe = collective.recipe.supervisor
-http-socket = unix
-user = searxer
-password = ohpleasedochangeme
-file = /tmp/supervisor.sock
-chmod = 0700
-programs =
-    50 searx    ${buildout:bin-directory}/searx-run
-
-
-[crontab_reboot]
-recipe = z3c.recipe.usercrontab
-times = @reboot
-command = ${buildout:bin-directory}/supervisord

requirements-dev.txt

@@ -0,0 +1,10 @@
+babel==2.2.0
+flake8==2.5.1
+mock==1.0.1
+nose2[coverage-plugin]
+plone.testing==4.0.15
+robotframework-selenium2library==1.7.4
+robotsuite==1.7.0
+transifex-client==0.11
+unittest2==1.1.0
+zope.testrunner==4.4.10

requirements.txt

@@ -1,12 +1,12 @@
-flask
-flask-babel
-requests
-lxml
-pyyaml
-pygments
-python-dateutil
-ndg-httpsclient
-pyopenssl
-pyasn1
-pyasn1-modules
-certifi
+certifi==2015.11.20.1
+flask==0.10.1
+flask-babel==0.9
+lxml==3.5.0
+ndg-httpsclient==0.4.0
+pyasn1==0.1.9
+pyasn1-modules==0.0.8
+pygments==2.0.2
+pyopenssl==0.15.1
+python-dateutil==2.4.2
+pyyaml==3.11
+requests==2.9.1

searx/engines/google.py

@@ -90,7 +90,7 @@ url_map = 'https://www.openstreetmap.org/'\
 search_path = '/search'
 search_url = ('https://{hostname}' +
               search_path +
-              '?{query}&start={offset}&gbv=1&gws_rd=cr')
+              '?{query}&start={offset}&gbv=1&gws_rd=ssl')
 
 # other URLs
 map_hostname_start = 'maps.google.'
@@ -99,7 +99,7 @@ redirect_path = '/url'
 images_path = '/images'
 
 # specific xpath variables
-results_xpath = '//li[@class="g"]'
+results_xpath = '//div[@class="g"]'
 url_xpath = './/h3/a/@href'
 title_xpath = './/h3'
 content_xpath = './/span[@class="st"]'

searx/testing.py

@@ -3,6 +3,7 @@
 
 from plone.testing import Layer
 from unittest2 import TestCase
+from os.path import dirname, join, abspath
 
 
 import os
@@ -42,11 +43,11 @@ class SearxRobotLayer(Layer):
             os.path.abspath(os.path.dirname(os.path.realpath(__file__))),
             'webapp.py'
         )
-        exe = os.path.abspath(os.path.dirname(__file__) + '/../bin/py')
+        exe = 'python'
 
         # set robot settings path
-        os.environ['SEARX_SETTINGS_PATH'] = os.path.abspath(
-            os.path.dirname(__file__) + '/settings_robot.yml')
+        os.environ['SEARX_SETTINGS_PATH'] = abspath(
+            dirname(__file__) + '/settings_robot.yml')
 
         # run the server
         self.server = subprocess.Popen(
@@ -68,3 +69,16 @@ class SearxTestCase(TestCase):
     """Base test case for non-robot tests."""
 
     layer = SearxTestLayer
+
+
+if __name__ == '__main__':
+    from tests.test_robot import test_suite
+    import sys
+    from zope.testrunner.runner import Runner
+
+    base_dir = abspath(join(dirname(__file__), '../tests'))
+    if sys.argv[1] == 'robot':
+        Runner(['--color',
+                '--auto-progress',
+                '--path', base_dir],
+               found_suites=[test_suite()]).run()

searx/tests/test_engines.py

@@ -1,47 +0,0 @@
-from searx.tests.engines.test_bing import *  # noqa
-from searx.tests.engines.test_bing_images import *  # noqa
-from searx.tests.engines.test_bing_news import *  # noqa
-from searx.tests.engines.test_blekko_images import *  # noqa
-from searx.tests.engines.test_btdigg import *  # noqa
-from searx.tests.engines.test_currency_convert import *  # noqa
-from searx.tests.engines.test_dailymotion import *  # noqa
-from searx.tests.engines.test_deezer import *  # noqa
-from searx.tests.engines.test_deviantart import *  # noqa
-from searx.tests.engines.test_digg import *  # noqa
-from searx.tests.engines.test_duckduckgo import *  # noqa
-from searx.tests.engines.test_duckduckgo_definitions import *  # noqa
-from searx.tests.engines.test_dummy import *  # noqa
-from searx.tests.engines.test_faroo import *  # noqa
-from searx.tests.engines.test_flickr import *  # noqa
-from searx.tests.engines.test_flickr_noapi import *  # noqa
-from searx.tests.engines.test_gigablast import *  # noqa
-from searx.tests.engines.test_github import *  # noqa
-from searx.tests.engines.test_google import *  # noqa
-from searx.tests.engines.test_google_images import *  # noqa
-from searx.tests.engines.test_google_news import *  # noqa
-from searx.tests.engines.test_kickass import *  # noqa
-from searx.tests.engines.test_mediawiki import *  # noqa
-from searx.tests.engines.test_mixcloud import *  # noqa
-from searx.tests.engines.test_openstreetmap import *  # noqa
-from searx.tests.engines.test_photon import *  # noqa
-from searx.tests.engines.test_piratebay import *  # noqa
-from searx.tests.engines.test_qwant import *  # noqa
-from searx.tests.engines.test_searchcode_code import *  # noqa
-from searx.tests.engines.test_searchcode_doc import *  # noqa
-from searx.tests.engines.test_soundcloud import *  # noqa
-from searx.tests.engines.test_spotify import *  # noqa
-from searx.tests.engines.test_stackoverflow import *  # noqa
-from searx.tests.engines.test_startpage import *  # noqa
-from searx.tests.engines.test_subtitleseeker import *  # noqa
-from searx.tests.engines.test_swisscows import *  # noqa
-from searx.tests.engines.test_twitter import *  # noqa
-from searx.tests.engines.test_vimeo import *  # noqa
-from searx.tests.engines.test_wolframalpha_api import *  # noqa
-from searx.tests.engines.test_wolframalpha_noapi import *  # noqa
-from searx.tests.engines.test_www1x import *  # noqa
-from searx.tests.engines.test_www500px import *  # noqa
-from searx.tests.engines.test_yacy import *  # noqa
-from searx.tests.engines.test_yahoo import *  # noqa
-from searx.tests.engines.test_youtube_api import *  # noqa
-from searx.tests.engines.test_youtube_noapi import *  # noqa
-from searx.tests.engines.test_yahoo_news import *  # noqa

setup.py

@@ -17,6 +17,8 @@ def read(*rnames):
 
 
 long_description = read('README.rst')
+requirements = map(str.strip, open('requirements.txt').readlines())
+dev_requirements = map(str.strip, open('requirements-dev.txt').readlines())
 
 setup(
     name='searx',
@@ -38,35 +40,9 @@ setup(
     license='GNU Affero General Public License',
     packages=find_packages('.'),
     zip_safe=False,
-    install_requires=[
-        'flask',
-        'flask-babel',
-        'requests',
-        'lxml',
-        'pyyaml',
-        'pygments',
-        'setuptools',
-        'python-dateutil',
-        'pyopenssl',
-        'ndg-httpsclient',
-        'pyasn1',
-        'pyasn1-modules',
-        'certifi'
-    ],
+    install_requires=requirements,
     extras_require={
-        'test': [
-            'coverage',
-            'flake8',
-            'mock',
-            'plone.testing',
-            'robotframework',
-            'robotframework-debuglibrary',
-            'robotframework-httplibrary',
-            'robotframework-selenium2library',
-            'robotsuite',
-            'unittest2',
-            'zope.testrunner',
-        ]
+        'test': dev_requirements
     },
     entry_points={
         'console_scripts': [
@@ -86,6 +62,9 @@ setup(
             'static/*/*/*/*/*.*',
             'templates/*/*.*',
             'templates/*/*/*.*',
+            'tests/*',
+            'tests/*/*',
+            'tests/*/*/*',
             'translations/*/*/*'
         ],
     },

tests/unit/engines/test_google.py

@@ -40,7 +40,7 @@ class TestGoogleEngine(SearxTestCase):
         self.assertEqual(google.response(response), [])
 
         html = """
-        <li class="g">
+        <div class="g">
             <h3 class="r">
                 <a href="http://this.should.be.the.link/">
                     <b>This</b> is <b>the</b> title
@@ -94,35 +94,35 @@ class TestGoogleEngine(SearxTestCase):
                     </a>
                 </div>
             </div>
-        </li>
-        <li class="g">
+        </div>
+        <div class="g">
             <h3 class="r">
                 <a href="http://www.google.com/images?q=toto">
                     <b>This</b>
                 </a>
             </h3>
-        </li>
-        <li class="g">
+        </div>
+        <div class="g">
             <h3 class="r">
                 <a href="http://www.google.com/search?q=toto">
                     <b>This</b> is
                 </a>
             </h3>
-        </li>
-        <li class="g">
+        </div>
+        <div class="g">
             <h3 class="r">
                 <a href="€">
                     <b>This</b> is <b>the</b>
                 </a>
             </h3>
-        </li>
-        <li class="g">
+        </div>
+        <div class="g">
             <h3 class="r">
                 <a href="/url?q=url">
                     <b>This</b> is <b>the</b>
                 </a>
             </h3>
-        </li>
+        </div>
         <p class="_Bmc" style="margin:3px 8px">
             <a href="/search?num=20&amp;safe=off&amp;q=t&amp;revid=1754833769&amp;sa=X&amp;ei=-&amp;ved=">
                 suggestion <b>title</b>

versions.cfg

@@ -1,122 +0,0 @@
-[versions]
-Babel = 1.3
-Flask = 0.10.1
-Flask-Babel = 0.9
-Jinja2 = 2.7.3
-MarkupSafe = 0.23
-Pygments = 2.0.2
-WebOb = 1.4.1
-WebTest = 2.0.18
-Werkzeug = 0.10.4
-collective.recipe.omelette = 0.16
-coverage = 3.7.1
-decorator = 3.4.2
-docutils = 0.12
-flake8 = 2.4.1
-itsdangerous = 0.24
-mccabe = 0.3.1
-mock = 1.0.1
-pep8 = 1.5.7
-plone.testing = 4.0.13
-pyflakes = 0.8.1
-pytz = 2015.4
-pyyaml = 3.11
-requests = 2.7.0
-robotframework-debuglibrary = 0.3
-robotframework-httplibrary = 0.4.2
-robotframework-selenium2library = 1.7.1
-robotsuite = 1.6.1
-selenium = 2.46.0
-speaklater = 1.3
-unittest2 = 1.0.1
-waitress = 0.8.9
-zc.recipe.testrunner = 2.0.0
-pyopenssl = 0.15.1
-ndg-httpsclient = 0.4.0
-pyasn1 = 0.1.8
-pyasn1-modules = 0.0.6
-certifi = 2015.11.20.1
-
-cffi = 1.1.2
-cryptography = 0.9.1
-
-# Required by:
-# robotsuite==1.6.1
-# searx==0.7.0
-lxml = 3.4.4
-
-# Required by:
-# searx==0.7.0
-python-dateutil = 2.4.2
-
-# Required by:
-# searx==0.7.0
-# zope.exceptions==4.0.7
-# zope.interface==4.1.2
-# zope.testrunner==4.4.9
-setuptools = 18.0.1
-
-# Required by:
-# WebTest==2.0.18
-beautifulsoup4 = 4.3.2
-
-# Required by:
-# cryptography==0.9.1
-enum34 = 1.0.4
-
-# Required by:
-# cryptography==0.9.1
-idna = 2.0
-
-# Required by:
-# cryptography==0.9.1
-ipaddress = 1.0.7
-
-# Required by:
-# robotframework-httplibrary==0.4.2
-jsonpatch = 1.11
-
-# Required by:
-# robotframework-httplibrary==0.4.2
-jsonpointer = 1.9
-
-# Required by:
-# traceback2==1.4.0
-linecache2 = 1.0.0
-
-# Required by:
-# cffi==1.1.2
-pycparser = 2.12
-
-# Required by:
-# robotframework-httplibrary==0.4.2
-robotframework = 2.8.7
-
-# Required by:
-# robotsuite==1.6.1
-# zope.testrunner==4.4.9
-six = 1.9.0
-
-# Required by:
-# unittest2==1.0.1
-traceback2 = 1.4.0
-
-# Required by:
-# collective.recipe.omelette==0.16
-zc.recipe.egg = 2.0.1
-
-# Required by:
-# zope.testrunner==4.4.9
-zope.exceptions = 4.0.7
-
-# Required by:
-# zope.testrunner==4.4.9
-zope.interface = 4.1.2
-
-# Required by:
-# plone.testing==4.0.13
-zope.testing = 4.2.0
-
-# Required by:
-# zc.recipe.testrunner==2.0.0
-zope.testrunner = 4.4.9