forked from Ponysearch/Ponysearch
Merge pull request #2352 from dalf/no_http
Remove HTTP connections as much as possible
This commit is contained in:
Alexandre Flament
GitHub
commit
925bb561a2
13 changed files with 28 additions and 114 deletions
|
|
@ -2,8 +2,8 @@
|
|||
|
||||
## Resources in the documentation
|
||||
|
||||
* [Development quickstart](http://searx.github.io/searx/dev/contribution_guide.html)
|
||||
* [Contribution guide](http://searx.github.io/searx/dev/contribution_guide.html)
|
||||
* [Development quickstart](https://searx.github.io/searx/dev/contribution_guide.html)
|
||||
* [Contribution guide](https://searx.github.io/searx/dev/contribution_guide.html)
|
||||
|
||||
## Submitting PRs
|
||||
|
||||
|
|
|
|||
|
|
@ -66,7 +66,7 @@ patch_searx_settings() {
|
|||
CONF="$1"
|
||||
|
||||
# Make sure that there is trailing slash at the end of BASE_URL
|
||||
# see http://www.gnu.org/savannah-checkouts/gnu/bash/manual/bash.html#Shell-Parameter-Expansion
|
||||
# see https://www.gnu.org/savannah-checkouts/gnu/bash/manual/bash.html#Shell-Parameter-Expansion
|
||||
export BASE_URL="${BASE_URL%/}/"
|
||||
|
||||
# update settings.yml
|
||||
|
|
|
|||
|
|
@ -67,7 +67,7 @@ to ``imgmath``:
|
|||
If your docs build (``make docs``) shows warnings like this::
|
||||
|
||||
WARNING: dot(1) not found, for better output quality install \
|
||||
graphviz from http://www.graphviz.org
|
||||
graphviz from https://www.graphviz.org
|
||||
..
|
||||
WARNING: LaTeX command 'latex' cannot be run (needed for math \
|
||||
display), check the imgmath_latex setting
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ Install with nginx
|
|||
.. _nginx server configuration:
|
||||
https://docs.nginx.com/nginx/admin-guide/web-server/web-server/#setting-up-virtual-servers
|
||||
.. _nginx beginners guide:
|
||||
http://nginx.org/en/docs/beginners_guide.html
|
||||
https://nginx.org/en/docs/beginners_guide.html
|
||||
.. _Getting Started wiki:
|
||||
https://www.nginx.com/resources/wiki/start/
|
||||
.. _uWSGI support from nginx:
|
||||
|
|
|
|||
|
|
@ -112,7 +112,7 @@ Global Settings
|
|||
specific instance of searx, a locale can be defined using an ISO language
|
||||
code, like ``fr``, ``en``, ``de``.
|
||||
|
||||
.. _requests proxies: http://requests.readthedocs.io/en/latest/user/advanced/#proxies
|
||||
.. _requests proxies: https://requests.readthedocs.io/en/latest/user/advanced/#proxies
|
||||
.. _PySocks: https://pypi.org/project/PySocks/
|
||||
|
||||
``proxies`` :
|
||||
|
|
|
|||
|
|
@ -49,11 +49,11 @@ extlinks['man'] = ('https://manpages.debian.org/jump?q=%s', '')
|
|||
#extlinks['role'] = (
|
||||
# 'https://www.sphinx-doc.org/en/master/usage/restructuredtext/roles.html#role-%s', '')
|
||||
extlinks['duref'] = (
|
||||
'http://docutils.sourceforge.net/docs/ref/rst/restructuredtext.html#%s', '')
|
||||
'https://docutils.sourceforge.net/docs/ref/rst/restructuredtext.html#%s', '')
|
||||
extlinks['durole'] = (
|
||||
'http://docutils.sourceforge.net/docs/ref/rst/roles.html#%s', '')
|
||||
'https://docutils.sourceforge.net/docs/ref/rst/roles.html#%s', '')
|
||||
extlinks['dudir'] = (
|
||||
'http://docutils.sourceforge.net/docs/ref/rst/directives.html#%s', '')
|
||||
'https://docutils.sourceforge.net/docs/ref/rst/directives.html#%s', '')
|
||||
extlinks['ctan'] = (
|
||||
'https://ctan.org/pkg/%s', 'CTAN: ')
|
||||
|
||||
|
|
|
|||
|
|
@ -117,8 +117,8 @@ Translation currently takes place on :ref:`transifex <translation>`.
|
|||
Documentation
|
||||
=============
|
||||
|
||||
.. _Sphinx: http://www.sphinx-doc.org
|
||||
.. _reST: http://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html
|
||||
.. _Sphinx: https://www.sphinx-doc.org
|
||||
.. _reST: https://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html
|
||||
|
||||
.. sidebar:: The reST sources
|
||||
|
||||
|
|
|
|||
|
|
@ -265,7 +265,7 @@ latitude latitude of result (in decimal format)
|
|||
longitude longitude of result (in decimal format)
|
||||
boundingbox boundingbox of result (array of 4. values
|
||||
``[lat-min, lat-max, lon-min, lon-max]``)
|
||||
geojson geojson of result (http://geojson.org)
|
||||
geojson geojson of result (https://geojson.org/)
|
||||
osm.type type of osm-object (if OSM-Result)
|
||||
osm.id id of osm-object (if OSM-Result)
|
||||
address.name name of object
|
||||
|
|
|
|||
|
|
@ -60,7 +60,7 @@ read :ref:`make test`.
|
|||
How to compile styles and javascript
|
||||
====================================
|
||||
|
||||
.. _less: http://lesscss.org/
|
||||
.. _less: https://lesscss.org/
|
||||
.. _NodeJS: https://nodejs.org
|
||||
|
||||
How to build styles
|
||||
|
|
|
|||
|
|
@ -1391,27 +1391,27 @@ The next example shows the difference of ``\tfrac`` (*textstyle*) and ``\dfrac``
|
|||
|
||||
.. _readability: https://docs.python-guide.org/writing/style/
|
||||
.. _Sphinx-Primer:
|
||||
http://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html
|
||||
https://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html
|
||||
.. _reST: https://docutils.sourceforge.io/rst.html
|
||||
.. _Sphinx Roles:
|
||||
https://www.sphinx-doc.org/en/master/usage/restructuredtext/roles.html
|
||||
.. _Sphinx: http://www.sphinx-doc.org
|
||||
.. _`sphinx-doc FAQ`: http://www.sphinx-doc.org/en/stable/faq.html
|
||||
.. _Sphinx: https://www.sphinx-doc.org
|
||||
.. _`sphinx-doc FAQ`: https://www.sphinx-doc.org/en/stable/faq.html
|
||||
.. _Sphinx markup constructs:
|
||||
http://www.sphinx-doc.org/en/stable/markup/index.html
|
||||
https://www.sphinx-doc.org/en/stable/markup/index.html
|
||||
.. _`sphinx cross references`:
|
||||
http://www.sphinx-doc.org/en/stable/markup/inline.html#cross-referencing-arbitrary-locations
|
||||
https://www.sphinx-doc.org/en/stable/markup/inline.html#cross-referencing-arbitrary-locations
|
||||
.. _sphinx.ext.extlinks:
|
||||
https://www.sphinx-doc.org/en/master/usage/extensions/extlinks.html
|
||||
.. _intersphinx: http://www.sphinx-doc.org/en/stable/ext/intersphinx.html
|
||||
.. _sphinx config: http://www.sphinx-doc.org/en/stable/config.html
|
||||
.. _Sphinx's autodoc: http://www.sphinx-doc.org/en/stable/ext/autodoc.html
|
||||
.. _intersphinx: https://www.sphinx-doc.org/en/stable/ext/intersphinx.html
|
||||
.. _sphinx config: https://www.sphinx-doc.org/en/stable/config.html
|
||||
.. _Sphinx's autodoc: https://www.sphinx-doc.org/en/stable/ext/autodoc.html
|
||||
.. _Sphinx's Python domain:
|
||||
http://www.sphinx-doc.org/en/stable/domains.html#the-python-domain
|
||||
https://www.sphinx-doc.org/en/stable/domains.html#the-python-domain
|
||||
.. _Sphinx's C domain:
|
||||
http://www.sphinx-doc.org/en/stable/domains.html#cross-referencing-c-constructs
|
||||
https://www.sphinx-doc.org/en/stable/domains.html#cross-referencing-c-constructs
|
||||
.. _doctree:
|
||||
http://www.sphinx-doc.org/en/master/extdev/tutorial.html?highlight=doctree#build-phases
|
||||
https://www.sphinx-doc.org/en/master/extdev/tutorial.html?highlight=doctree#build-phases
|
||||
.. _docutils: http://docutils.sourceforge.net/docs/index.html
|
||||
.. _docutils FAQ: http://docutils.sourceforge.net/FAQ.html
|
||||
.. _linuxdoc: https://return42.github.io/linuxdoc
|
||||
|
|
@ -1424,5 +1424,5 @@ The next example shows the difference of ``\tfrac`` (*textstyle*) and ``\dfrac``
|
|||
.. _ImageMagick: https://www.imagemagick.org
|
||||
|
||||
.. _`Emacs Table Mode`: https://www.emacswiki.org/emacs/TableMode
|
||||
.. _`Online Tables Generator`: http://www.tablesgenerator.com/text_tables
|
||||
.. _`Online Tables Generator`: https://www.tablesgenerator.com/text_tables
|
||||
.. _`OASIS XML Exchange Table Model`: https://www.oasis-open.org/specs/tm9901.html
|
||||
|
|
|
|||
|
|
@ -113,7 +113,7 @@ def searx_bang(full_query):
|
|||
|
||||
def dbpedia(query, lang):
|
||||
# dbpedia autocompleter, no HTTPS
|
||||
autocomplete_url = 'http://lookup.dbpedia.org/api/search.asmx/KeywordSearch?'
|
||||
autocomplete_url = 'https://lookup.dbpedia.org/api/search.asmx/KeywordSearch?'
|
||||
|
||||
response = get(autocomplete_url + urlencode(dict(QueryString=query)))
|
||||
|
||||
|
|
@ -121,8 +121,7 @@ def dbpedia(query, lang):
|
|||
|
||||
if response.ok:
|
||||
dom = etree.fromstring(response.content)
|
||||
results = dom.xpath('//a:Result/a:Label//text()',
|
||||
namespaces={'a': 'http://lookup.dbpedia.org/'})
|
||||
results = dom.xpath('//Result/Label//text()')
|
||||
|
||||
return results
|
||||
|
||||
|
|
|
|||
|
|
@ -1,85 +0,0 @@
|
|||
from html.parser import HTMLParser
|
||||
from urllib.parse import urlencode
|
||||
|
||||
|
||||
url = 'http://www.filecrop.com/'
|
||||
search_url = url + '/search.php?{query}&size_i=0&size_f=100000000&engine_r=1&engine_d=1&engine_e=1&engine_4=1&engine_m=1&pos={index}' # noqa
|
||||
|
||||
paging = True
|
||||
|
||||
|
||||
class FilecropResultParser(HTMLParser): # pylint: disable=W0223 # (see https://bugs.python.org/issue31844)
|
||||
|
||||
def __init__(self):
|
||||
HTMLParser.__init__(self)
|
||||
self.__start_processing = False
|
||||
|
||||
self.results = []
|
||||
self.result = {}
|
||||
|
||||
self.tr_counter = 0
|
||||
self.data_counter = 0
|
||||
|
||||
def handle_starttag(self, tag, attrs):
|
||||
|
||||
if tag == 'tr':
|
||||
if ('bgcolor', '#edeff5') in attrs or\
|
||||
('bgcolor', '#ffffff') in attrs:
|
||||
self.__start_processing = True
|
||||
|
||||
if not self.__start_processing:
|
||||
return
|
||||
|
||||
if tag == 'label':
|
||||
self.result['title'] = [attr[1] for attr in attrs
|
||||
if attr[0] == 'title'][0]
|
||||
elif tag == 'a' and ('rel', 'nofollow') in attrs\
|
||||
and ('class', 'sourcelink') in attrs:
|
||||
if 'content' in self.result:
|
||||
self.result['content'] += [attr[1] for attr in attrs
|
||||
if attr[0] == 'title'][0]
|
||||
else:
|
||||
self.result['content'] = [attr[1] for attr in attrs
|
||||
if attr[0] == 'title'][0]
|
||||
self.result['content'] += ' '
|
||||
elif tag == 'a':
|
||||
self.result['url'] = url + [attr[1] for attr in attrs
|
||||
if attr[0] == 'href'][0]
|
||||
|
||||
def handle_endtag(self, tag):
|
||||
if self.__start_processing is False:
|
||||
return
|
||||
|
||||
if tag == 'tr':
|
||||
self.tr_counter += 1
|
||||
|
||||
if self.tr_counter == 2:
|
||||
self.__start_processing = False
|
||||
self.tr_counter = 0
|
||||
self.data_counter = 0
|
||||
self.results.append(self.result)
|
||||
self.result = {}
|
||||
|
||||
def handle_data(self, data):
|
||||
if not self.__start_processing:
|
||||
return
|
||||
|
||||
if 'content' in self.result:
|
||||
self.result['content'] += data + ' '
|
||||
else:
|
||||
self.result['content'] = data + ' '
|
||||
|
||||
self.data_counter += 1
|
||||
|
||||
|
||||
def request(query, params):
|
||||
index = 1 + (params['pageno'] - 1) * 30
|
||||
params['url'] = search_url.format(query=urlencode({'w': query}), index=index)
|
||||
return params
|
||||
|
||||
|
||||
def response(resp):
|
||||
parser = FilecropResultParser()
|
||||
parser.feed(resp.text)
|
||||
|
||||
return parser.results
|
||||
|
|
@ -252,7 +252,7 @@ pyenv-python: pyenv-install
|
|||
# PyPi is required and since uploads via setuptools is not recommended, we have
|
||||
# to imstall / use twine ... its really a mess.
|
||||
#
|
||||
# [1] http://python-packaging.readthedocs.io/en/latest/dependencies.html#packages-not-on-pypi
|
||||
# [1] https://python-packaging.readthedocs.io/en/latest/dependencies.html#packages-not-on-pypi
|
||||
# [2] https://github.com/pypa/pip/pull/1519
|
||||
|
||||
# https://github.com/pypa/twine
|
||||
|
|
|
|||
Loading…
Reference in a new issue