forked from Ponysearch/Ponysearch
Alexandre Flament
GitHub
commit
c6922ae7c5
187 changed files with 2967 additions and 2918 deletions
7
Makefile
7
Makefile
|
|
@ -54,8 +54,8 @@ search.checker.%: install
|
|||
$(Q)./manage pyenv.cmd searx-checker -v "$(subst _, ,$(patsubst search.checker.%,%,$@))"
|
||||
|
||||
PHONY += test ci.test test.shell
|
||||
ci.test: test.yamllint test.pep8 test.pylint test.unit test.robot
|
||||
test: test.yamllint test.pep8 test.pylint test.unit test.robot test.shell
|
||||
ci.test: test.yamllint test.black test.pylint test.unit test.robot
|
||||
test: test.yamllint test.black test.pylint test.unit test.robot test.shell
|
||||
test.shell:
|
||||
$(Q)shellcheck -x -s dash \
|
||||
dockerfiles/docker-entrypoint.sh
|
||||
|
|
@ -88,7 +88,8 @@ MANAGE += node.env node.clean
|
|||
MANAGE += py.build py.clean
|
||||
MANAGE += pyenv pyenv.install pyenv.uninstall
|
||||
MANAGE += pypi.upload pypi.upload.test
|
||||
MANAGE += test.yamllint test.pylint test.pep8 test.unit test.coverage test.robot test.clean
|
||||
MANAGE += format.python
|
||||
MANAGE += test.yamllint test.pylint test.black test.unit test.coverage test.robot test.clean
|
||||
MANAGE += themes.all themes.oscar themes.simple themes.simple.test pygments.less
|
||||
MANAGE += static.build.commit static.build.drop static.build.restore
|
||||
MANAGE += nvm.install nvm.clean nvm.status nvm.nodejs
|
||||
|
|
|
|||
27
manage
27
manage
|
|
@ -24,6 +24,8 @@ PY_SETUP_EXTRAS='[test]'
|
|||
GECKODRIVER_VERSION="v0.30.0"
|
||||
export NODE_MINIMUM_VERSION="16.13.0"
|
||||
# SPHINXOPTS=
|
||||
BLACK_OPTIONS=("--target-version" "py37" "--line-length" "120" "--skip-string-normalization")
|
||||
BLACK_TARGETS=("--exclude" "searx/static,searx/languages.py" "searx" "searxng_extra" "tests")
|
||||
|
||||
pylint.FILES() {
|
||||
|
||||
|
|
@ -31,8 +33,7 @@ pylint.FILES() {
|
|||
#
|
||||
# # lint: pylint
|
||||
#
|
||||
# These py files are linted by test.pylint(), all other files are linted by
|
||||
# test.pep8()
|
||||
# These py files are linted by test.pylint()
|
||||
|
||||
grep -l -r --include \*.py '^#[[:blank:]]*lint:[[:blank:]]*pylint' searx searxng_extra tests
|
||||
}
|
||||
|
|
@ -89,10 +90,12 @@ pyenv.:
|
|||
OK : test if virtualenv is OK
|
||||
pypi.upload:
|
||||
Upload python packages to PyPi (to test use pypi.upload.test)
|
||||
format.:
|
||||
python : format Python code source using black
|
||||
test.:
|
||||
yamllint : lint YAML files (YAMLLINT_FILES)
|
||||
pylint : lint PYLINT_FILES, searx/engines, searx & tests
|
||||
pep8 : pycodestyle (pep8) for all files except PYLINT_FILES
|
||||
black : check black code format
|
||||
unit : run unit tests
|
||||
coverage : run unit tests with coverage
|
||||
robot : run robot test
|
||||
|
|
@ -617,6 +620,12 @@ pypi.upload.test() {
|
|||
pyenv.cmd twine upload -r testpypi "${PYDIST}"/*
|
||||
}
|
||||
|
||||
format.python() {
|
||||
build_msg TEST "[format.python] black \$BLACK_TARGETS"
|
||||
pyenv.cmd black "${BLACK_OPTIONS[@]}" "${BLACK_TARGETS[@]}"
|
||||
dump_return $?
|
||||
}
|
||||
|
||||
test.yamllint() {
|
||||
build_msg TEST "[yamllint] \$YAMLLINT_FILES"
|
||||
pyenv.cmd yamllint --format parsable "${YAMLLINT_FILES[@]}"
|
||||
|
|
@ -646,15 +655,9 @@ test.pylint() {
|
|||
dump_return $?
|
||||
}
|
||||
|
||||
test.pep8() {
|
||||
build_msg TEST 'pycodestyle (formerly pep8)'
|
||||
local _exclude=""
|
||||
printf -v _exclude '%s, ' "${PYLINT_FILES[@]}"
|
||||
pyenv.cmd pycodestyle \
|
||||
--exclude="searx/static, searx/languages.py, $_exclude " \
|
||||
--max-line-length=120 \
|
||||
--ignore "E117,E252,E402,E722,E741,W503,W504,W605" \
|
||||
searx tests
|
||||
test.black() {
|
||||
build_msg TEST "[black] \$BLACK_TARGETS"
|
||||
pyenv.cmd black --check --diff "${BLACK_OPTIONS[@]}" "${BLACK_TARGETS[@]}"
|
||||
dump_return $?
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
mock==4.0.3
|
||||
nose2[coverage_plugin]==0.10.0
|
||||
cov-core==1.15.0
|
||||
black==21.12b0
|
||||
pycodestyle==2.8.0
|
||||
pylint==2.12.2
|
||||
splinter==0.17.0
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ if settings is not None:
|
|||
|
||||
_unset = object()
|
||||
|
||||
|
||||
def get_setting(name, default=_unset):
|
||||
"""Returns the value to which ``name`` point. If there is no such name in the
|
||||
settings and the ``default`` is unset, a :py:obj:`KeyError` is raised.
|
||||
|
|
@ -80,14 +81,9 @@ def logging_config_debug():
|
|||
'levelname': {'color': 8},
|
||||
'name': {'color': 8},
|
||||
'programname': {'color': 'cyan'},
|
||||
'username': {'color': 'yellow'}
|
||||
'username': {'color': 'yellow'},
|
||||
}
|
||||
coloredlogs.install(
|
||||
level=log_level,
|
||||
level_styles=level_styles,
|
||||
field_styles=field_styles,
|
||||
fmt=LOG_FORMAT_DEBUG
|
||||
)
|
||||
coloredlogs.install(level=log_level, level_styles=level_styles, field_styles=field_styles, fmt=LOG_FORMAT_DEBUG)
|
||||
else:
|
||||
logging.basicConfig(level=logging.getLevelName(log_level), format=LOG_FORMAT_DEBUG)
|
||||
|
||||
|
|
|
|||
|
|
@ -13,8 +13,7 @@ random_string_letters = string.ascii_lowercase + string.digits + string.ascii_up
|
|||
|
||||
|
||||
def random_characters():
|
||||
return [random.choice(random_string_letters)
|
||||
for _ in range(random.randint(8, 32))]
|
||||
return [random.choice(random_string_letters) for _ in range(random.randint(8, 32))]
|
||||
|
||||
|
||||
def random_string():
|
||||
|
|
@ -39,11 +38,13 @@ def random_uuid():
|
|||
return str(uuid.uuid4())
|
||||
|
||||
|
||||
random_types = {'string': random_string,
|
||||
random_types = {
|
||||
'string': random_string,
|
||||
'int': random_int,
|
||||
'float': random_float,
|
||||
'sha256': random_sha256,
|
||||
'uuid': random_uuid}
|
||||
'uuid': random_uuid,
|
||||
}
|
||||
|
||||
|
||||
# required answerer function
|
||||
|
|
@ -62,6 +63,8 @@ def answer(query):
|
|||
# required answerer function
|
||||
# returns information about the answerer
|
||||
def self_info():
|
||||
return {'name': gettext('Random value generator'),
|
||||
return {
|
||||
'name': gettext('Random value generator'),
|
||||
'description': gettext('Generate different random values'),
|
||||
'examples': ['random {}'.format(x) for x in random_types]}
|
||||
'examples': ['random {}'.format(x) for x in random_types],
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,11 +4,7 @@ from operator import mul
|
|||
from flask_babel import gettext
|
||||
|
||||
|
||||
keywords = ('min',
|
||||
'max',
|
||||
'avg',
|
||||
'sum',
|
||||
'prod')
|
||||
keywords = ('min', 'max', 'avg', 'sum', 'prod')
|
||||
|
||||
|
||||
# required answerer function
|
||||
|
|
@ -47,6 +43,8 @@ def answer(query):
|
|||
# required answerer function
|
||||
# returns information about the answerer
|
||||
def self_info():
|
||||
return {'name': gettext('Statistics functions'),
|
||||
return {
|
||||
'name': gettext('Statistics functions'),
|
||||
'description': gettext('Compute {functions} of the arguments').format(functions='/'.join(keywords)),
|
||||
'examples': ['avg 123 548 2.04 24.2']}
|
||||
'examples': ['avg 123 548 2.04 24.2'],
|
||||
}
|
||||
|
|
|
|||
|
|
@ -120,13 +120,14 @@ def wikipedia(query, lang):
|
|||
return []
|
||||
|
||||
|
||||
backends = {'dbpedia': dbpedia,
|
||||
backends = {
|
||||
'dbpedia': dbpedia,
|
||||
'duckduckgo': duckduckgo,
|
||||
'google': google,
|
||||
'startpage': startpage,
|
||||
'swisscows': swisscows,
|
||||
'qwant': qwant,
|
||||
'wikipedia': wikipedia
|
||||
'wikipedia': wikipedia,
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -23,10 +23,12 @@ from pathlib import Path
|
|||
|
||||
data_dir = Path(__file__).parent
|
||||
|
||||
|
||||
def _load(filename):
|
||||
with open(data_dir / filename, encoding='utf-8') as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
def ahmia_blacklist_loader():
|
||||
"""Load data from `ahmia_blacklist.txt` and return a list of MD5 values of onion
|
||||
names. The MD5 values are fetched by::
|
||||
|
|
@ -39,6 +41,7 @@ def ahmia_blacklist_loader():
|
|||
with open(str(data_dir / 'ahmia_blacklist.txt'), encoding='utf-8') as f:
|
||||
return f.read().split()
|
||||
|
||||
|
||||
ENGINES_LANGUAGES = _load('engines_languages.json')
|
||||
CURRENCIES = _load('currencies.json')
|
||||
USER_AGENTS = _load('useragents.json')
|
||||
|
|
|
|||
|
|
@ -43,11 +43,15 @@ def response(resp):
|
|||
filesize, filesize_multiplier = filesize_info.split()
|
||||
filesize = get_torrent_size(filesize, filesize_multiplier)
|
||||
|
||||
results.append({'url': href,
|
||||
results.append(
|
||||
{
|
||||
'url': href,
|
||||
'title': title,
|
||||
'seed': seed,
|
||||
'leech': leech,
|
||||
'filesize': filesize,
|
||||
'template': 'torrent.html'})
|
||||
'template': 'torrent.html',
|
||||
}
|
||||
)
|
||||
|
||||
return results
|
||||
|
|
|
|||
|
|
@ -57,6 +57,7 @@ engine_shortcuts = {}
|
|||
|
||||
"""
|
||||
|
||||
|
||||
def load_engine(engine_data):
|
||||
"""Load engine from ``engine_data``.
|
||||
|
||||
|
|
@ -166,20 +167,19 @@ def set_language_attributes(engine):
|
|||
# settings.yml
|
||||
if engine.language not in engine.supported_languages:
|
||||
raise ValueError(
|
||||
"settings.yml - engine: '%s' / language: '%s' not supported" % (
|
||||
engine.name, engine.language ))
|
||||
"settings.yml - engine: '%s' / language: '%s' not supported" % (engine.name, engine.language)
|
||||
)
|
||||
|
||||
if isinstance(engine.supported_languages, dict):
|
||||
engine.supported_languages = {
|
||||
engine.language : engine.supported_languages[engine.language]
|
||||
}
|
||||
engine.supported_languages = {engine.language: engine.supported_languages[engine.language]}
|
||||
else:
|
||||
engine.supported_languages = [engine.language]
|
||||
|
||||
# find custom aliases for non standard language codes
|
||||
for engine_lang in engine.supported_languages:
|
||||
iso_lang = match_language(engine_lang, BABEL_LANGS, fallback=None)
|
||||
if (iso_lang
|
||||
if (
|
||||
iso_lang
|
||||
and iso_lang != engine_lang
|
||||
and not engine_lang.startswith(iso_lang)
|
||||
and iso_lang not in engine.supported_languages
|
||||
|
|
@ -197,14 +197,12 @@ def set_language_attributes(engine):
|
|||
}
|
||||
engine.fetch_supported_languages = (
|
||||
# pylint: disable=protected-access
|
||||
lambda: engine._fetch_supported_languages(
|
||||
get(engine.supported_languages_url, headers=headers))
|
||||
lambda: engine._fetch_supported_languages(get(engine.supported_languages_url, headers=headers))
|
||||
)
|
||||
|
||||
|
||||
def update_attributes_for_tor(engine):
|
||||
if (settings['outgoing'].get('using_tor_proxy')
|
||||
and hasattr(engine, 'onion_url') ):
|
||||
if settings['outgoing'].get('using_tor_proxy') and hasattr(engine, 'onion_url'):
|
||||
engine.search_url = engine.onion_url + getattr(engine, 'search_path', '')
|
||||
engine.timeout += settings['outgoing'].get('extra_proxy_timeout', 0)
|
||||
|
||||
|
|
@ -217,9 +215,7 @@ def is_missing_required_attributes(engine):
|
|||
missing = False
|
||||
for engine_attr in dir(engine):
|
||||
if not engine_attr.startswith('_') and getattr(engine, engine_attr) is None:
|
||||
logger.error(
|
||||
'Missing engine config attribute: "{0}.{1}"'
|
||||
.format(engine.name, engine_attr))
|
||||
logger.error('Missing engine config attribute: "{0}.{1}"'.format(engine.name, engine_attr))
|
||||
missing = True
|
||||
return missing
|
||||
|
||||
|
|
@ -230,8 +226,7 @@ def is_engine_active(engine):
|
|||
return False
|
||||
|
||||
# exclude onion engines if not using tor
|
||||
if ('onions' in engine.categories
|
||||
and not settings['outgoing'].get('using_tor_proxy') ):
|
||||
if 'onions' in engine.categories and not settings['outgoing'].get('using_tor_proxy'):
|
||||
return False
|
||||
|
||||
return True
|
||||
|
|
@ -253,8 +248,7 @@ def register_engine(engine):
|
|||
|
||||
|
||||
def load_engines(engine_list):
|
||||
"""usage: ``engine_list = settings['engines']``
|
||||
"""
|
||||
"""usage: ``engine_list = settings['engines']``"""
|
||||
engines.clear()
|
||||
engine_shortcuts.clear()
|
||||
categories.clear()
|
||||
|
|
|
|||
|
|
@ -25,9 +25,7 @@ page_size = 10
|
|||
# search url
|
||||
search_url = 'http://juhanurmihxlp77nkq76byazcldy2hlmovfu2epvl5ankdibsot4csyd.onion/search/?{query}'
|
||||
time_range_support = True
|
||||
time_range_dict = {'day': 1,
|
||||
'week': 7,
|
||||
'month': 30}
|
||||
time_range_dict = {'day': 1, 'week': 7, 'month': 30}
|
||||
|
||||
# xpaths
|
||||
results_xpath = '//li[@class="result"]'
|
||||
|
|
@ -65,10 +63,7 @@ def response(resp):
|
|||
title = extract_text(eval_xpath(result, title_xpath))
|
||||
content = extract_text(eval_xpath(result, content_xpath))
|
||||
|
||||
results.append({'url': cleaned_url,
|
||||
'title': title,
|
||||
'content': content,
|
||||
'is_onion': True})
|
||||
results.append({'url': cleaned_url, 'title': title, 'content': content, 'is_onion': True})
|
||||
|
||||
# get spelling corrections
|
||||
for correction in eval_xpath_list(dom, correction_xpath):
|
||||
|
|
|
|||
|
|
@ -55,11 +55,7 @@ def response(resp):
|
|||
url = base_url + link.attrib.get('href') + '#downloads'
|
||||
title = extract_text(link)
|
||||
img_src = base_url + eval_xpath_getindex(result, './/img/@src', 0)
|
||||
res = {
|
||||
'url': url,
|
||||
'title': title,
|
||||
'img_src': img_src
|
||||
}
|
||||
res = {'url': url, 'title': title, 'img_src': img_src}
|
||||
|
||||
results.append(res)
|
||||
|
||||
|
|
|
|||
|
|
@ -39,6 +39,7 @@ def locale_to_lang_code(locale):
|
|||
# wikis for some languages were moved off from the main site, we need to make
|
||||
# requests to correct URLs to be able to get results in those languages
|
||||
lang_urls = {
|
||||
# fmt: off
|
||||
'all': {
|
||||
'base': 'https://wiki.archlinux.org',
|
||||
'search': '/index.php?title=Special:Search&offset={offset}&{query}'
|
||||
|
|
@ -63,6 +64,7 @@ lang_urls = {
|
|||
'base': 'http://archtr.org/wiki',
|
||||
'search': '/index.php?title=Özel:Ara&offset={offset}&{query}'
|
||||
}
|
||||
# fmt: on
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -95,7 +97,7 @@ main_langs = {
|
|||
'sl': 'Slovenský',
|
||||
'th': 'ไทย',
|
||||
'uk': 'Українська',
|
||||
'zh': '简体中文'
|
||||
'zh': '简体中文',
|
||||
}
|
||||
supported_languages = dict(lang_urls, **main_langs)
|
||||
|
||||
|
|
@ -139,7 +141,6 @@ def response(resp):
|
|||
href = urljoin(base_url, link.attrib.get('href'))
|
||||
title = extract_text(link)
|
||||
|
||||
results.append({'url': href,
|
||||
'title': title})
|
||||
results.append({'url': href, 'title': title})
|
||||
|
||||
return results
|
||||
|
|
|
|||
|
|
@ -27,19 +27,23 @@ nb_per_page = 20
|
|||
search_api = 'https://api.artic.edu/api/v1/artworks/search?'
|
||||
image_api = 'https://www.artic.edu/iiif/2/'
|
||||
|
||||
|
||||
def request(query, params):
|
||||
|
||||
args = urlencode({
|
||||
args = urlencode(
|
||||
{
|
||||
'q': query,
|
||||
'page': params['pageno'],
|
||||
'fields': 'id,title,artist_display,medium_display,image_id,date_display,dimensions,artist_titles',
|
||||
'limit': nb_per_page,
|
||||
})
|
||||
}
|
||||
)
|
||||
params['url'] = search_api + args
|
||||
|
||||
logger.debug("query_url --> %s", params['url'])
|
||||
return params
|
||||
|
||||
|
||||
def response(resp):
|
||||
|
||||
results = []
|
||||
|
|
@ -50,14 +54,16 @@ def response(resp):
|
|||
if not result['image_id']:
|
||||
continue
|
||||
|
||||
results.append({
|
||||
results.append(
|
||||
{
|
||||
'url': 'https://artic.edu/artworks/%(id)s' % result,
|
||||
'title': result['title'] + " (%(date_display)s) // %(artist_display)s" % result,
|
||||
'content': result['medium_display'],
|
||||
'author': ', '.join(result['artist_titles']),
|
||||
'img_src': image_api + '/%(image_id)s/full/843,/0/default.jpg' % result,
|
||||
'img_format': result['dimensions'],
|
||||
'template': 'images.html'
|
||||
})
|
||||
'template': 'images.html',
|
||||
}
|
||||
)
|
||||
|
||||
return results
|
||||
|
|
|
|||
|
|
@ -20,8 +20,9 @@ about = {
|
|||
categories = ['science']
|
||||
paging = True
|
||||
|
||||
base_url = 'https://export.arxiv.org/api/query?search_query=all:'\
|
||||
+ '{query}&start={offset}&max_results={number_of_results}'
|
||||
base_url = (
|
||||
'https://export.arxiv.org/api/query?search_query=all:' + '{query}&start={offset}&max_results={number_of_results}'
|
||||
)
|
||||
|
||||
# engine dependent config
|
||||
number_of_results = 10
|
||||
|
|
@ -31,9 +32,7 @@ def request(query, params):
|
|||
# basic search
|
||||
offset = (params['pageno'] - 1) * number_of_results
|
||||
|
||||
string_args = dict(query=query,
|
||||
offset=offset,
|
||||
number_of_results=number_of_results)
|
||||
string_args = dict(query=query, offset=offset, number_of_results=number_of_results)
|
||||
|
||||
params['url'] = base_url.format(**string_args)
|
||||
|
||||
|
|
@ -65,10 +64,7 @@ def response(resp):
|
|||
|
||||
publishedDate = datetime.strptime(eval_xpath_getindex(entry, './/published', 0).text, '%Y-%m-%dT%H:%M:%SZ')
|
||||
|
||||
res_dict = {'url': url,
|
||||
'title': title,
|
||||
'publishedDate': publishedDate,
|
||||
'content': content}
|
||||
res_dict = {'url': url, 'title': title, 'publishedDate': publishedDate, 'content': content}
|
||||
|
||||
results.append(res_dict)
|
||||
|
||||
|
|
|
|||
|
|
@ -44,9 +44,7 @@ def request(query, params):
|
|||
pageno : 1 # number of the requested page
|
||||
'''
|
||||
|
||||
search_path = search_string.format(
|
||||
query=urlencode({'q': query}),
|
||||
page=params['pageno'])
|
||||
search_path = search_string.format(query=urlencode({'q': query}), page=params['pageno'])
|
||||
|
||||
params['url'] = base_url + search_path
|
||||
|
||||
|
|
|
|||
|
|
@ -21,8 +21,10 @@ about = {
|
|||
|
||||
categories = ['science']
|
||||
|
||||
base_url = 'https://api.base-search.net/cgi-bin/BaseHttpSearchInterface.fcgi'\
|
||||
base_url = (
|
||||
'https://api.base-search.net/cgi-bin/BaseHttpSearchInterface.fcgi'
|
||||
+ '?func=PerformSearch&{query}&boost=oa&hits={hits}&offset={offset}'
|
||||
)
|
||||
|
||||
# engine dependent config
|
||||
paging = True
|
||||
|
|
@ -47,7 +49,7 @@ shorcut_dict = {
|
|||
'source:': 'dcsource:',
|
||||
'subject:': 'dcsubject:',
|
||||
'title:': 'dctitle:',
|
||||
'type:': 'dcdctype:'
|
||||
'type:': 'dcdctype:',
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -59,9 +61,7 @@ def request(query, params):
|
|||
# basic search
|
||||
offset = (params['pageno'] - 1) * number_of_results
|
||||
|
||||
string_args = dict(query=urlencode({'query': query}),
|
||||
offset=offset,
|
||||
hits=number_of_results)
|
||||
string_args = dict(query=urlencode({'query': query}), offset=offset, hits=number_of_results)
|
||||
|
||||
params['url'] = base_url.format(**string_args)
|
||||
|
||||
|
|
@ -103,14 +103,9 @@ def response(resp):
|
|||
pass
|
||||
|
||||
if publishedDate is not None:
|
||||
res_dict = {'url': url,
|
||||
'title': title,
|
||||
'publishedDate': publishedDate,
|
||||
'content': content}
|
||||
res_dict = {'url': url, 'title': title, 'publishedDate': publishedDate, 'content': content}
|
||||
else:
|
||||
res_dict = {'url': url,
|
||||
'title': title,
|
||||
'content': content}
|
||||
res_dict = {'url': url, 'title': title, 'content': content}
|
||||
|
||||
results.append(res_dict)
|
||||
|
||||
|
|
|
|||
|
|
@ -36,9 +36,11 @@ inital_query = 'search?{query}&search=&form=QBLH'
|
|||
# following queries: https://www.bing.com/search?q=foo&search=&first=11&FORM=PERE
|
||||
page_query = 'search?{query}&search=&first={offset}&FORM=PERE'
|
||||
|
||||
|
||||
def _get_offset_from_pageno(pageno):
|
||||
return (pageno - 1) * 10 + 1
|
||||
|
||||
|
||||
def request(query, params):
|
||||
|
||||
offset = _get_offset_from_pageno(params.get('pageno', 1))
|
||||
|
|
@ -53,17 +55,11 @@ def request(query, params):
|
|||
if params['language'] == 'all':
|
||||
lang = 'EN'
|
||||
else:
|
||||
lang = match_language(
|
||||
params['language'], supported_languages, language_aliases
|
||||
)
|
||||
lang = match_language(params['language'], supported_languages, language_aliases)
|
||||
|
||||
query = 'language:{} {}'.format(
|
||||
lang.split('-')[0].upper(), query
|
||||
)
|
||||
query = 'language:{} {}'.format(lang.split('-')[0].upper(), query)
|
||||
|
||||
search_path = search_string.format(
|
||||
query = urlencode({'q': query}),
|
||||
offset = offset)
|
||||
search_path = search_string.format(query=urlencode({'q': query}), offset=offset)
|
||||
|
||||
if offset > 1:
|
||||
referer = base_url + inital_query.format(query=urlencode({'q': query}))
|
||||
|
|
@ -72,11 +68,10 @@ def request(query, params):
|
|||
|
||||
params['url'] = base_url + search_path
|
||||
params['headers']['Accept-Language'] = "en-US,en;q=0.5"
|
||||
params['headers']['Accept'] = (
|
||||
'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
|
||||
)
|
||||
params['headers']['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
|
||||
return params
|
||||
|
||||
|
||||
def response(resp):
|
||||
|
||||
results = []
|
||||
|
|
@ -95,11 +90,7 @@ def response(resp):
|
|||
content = extract_text(eval_xpath(result, './/p'))
|
||||
|
||||
# append result
|
||||
results.append({
|
||||
'url': url,
|
||||
'title': title,
|
||||
'content': content
|
||||
})
|
||||
results.append({'url': url, 'title': title, 'content': content})
|
||||
|
||||
# parse results again if nothing is found yet
|
||||
for result in eval_xpath(dom, '//li[@class="b_algo"]'):
|
||||
|
|
@ -110,11 +101,7 @@ def response(resp):
|
|||
content = extract_text(eval_xpath(result, './/p'))
|
||||
|
||||
# append result
|
||||
results.append({
|
||||
'url': url,
|
||||
'title': title,
|
||||
'content': content
|
||||
})
|
||||
results.append({'url': url, 'title': title, 'content': content})
|
||||
|
||||
try:
|
||||
result_len_container = "".join(eval_xpath(dom, '//span[@class="sb_count"]//text()'))
|
||||
|
|
|
|||
|
|
@ -6,10 +6,13 @@
|
|||
from urllib.parse import urlencode
|
||||
from lxml import html
|
||||
from json import loads
|
||||
from searx.utils import match_language
|
||||
|
||||
from searx.utils import match_language
|
||||
from searx.engines.bing import language_aliases
|
||||
from searx.engines.bing import _fetch_supported_languages, supported_languages_url # NOQA # pylint: disable=unused-import
|
||||
from searx.engines.bing import ( # pylint: disable=unused-import
|
||||
_fetch_supported_languages,
|
||||
supported_languages_url,
|
||||
)
|
||||
|
||||
# about
|
||||
about = {
|
||||
|
|
@ -31,39 +34,33 @@ number_of_results = 28
|
|||
|
||||
# search-url
|
||||
base_url = 'https://www.bing.com/'
|
||||
search_string = 'images/search'\
|
||||
'?{query}'\
|
||||
'&count={count}'\
|
||||
'&first={first}'\
|
||||
search_string = (
|
||||
# fmt: off
|
||||
'images/search'
|
||||
'?{query}'
|
||||
'&count={count}'
|
||||
'&first={first}'
|
||||
'&tsc=ImageHoverTitle'
|
||||
# fmt: on
|
||||
)
|
||||
time_range_string = '&qft=+filterui:age-lt{interval}'
|
||||
time_range_dict = {'day': '1440',
|
||||
'week': '10080',
|
||||
'month': '43200',
|
||||
'year': '525600'}
|
||||
time_range_dict = {'day': '1440', 'week': '10080', 'month': '43200', 'year': '525600'}
|
||||
|
||||
# safesearch definitions
|
||||
safesearch_types = {2: 'STRICT',
|
||||
1: 'DEMOTE',
|
||||
0: 'OFF'}
|
||||
safesearch_types = {2: 'STRICT', 1: 'DEMOTE', 0: 'OFF'}
|
||||
|
||||
|
||||
# do search-request
|
||||
def request(query, params):
|
||||
offset = ((params['pageno'] - 1) * number_of_results) + 1
|
||||
|
||||
search_path = search_string.format(
|
||||
query=urlencode({'q': query}),
|
||||
count=number_of_results,
|
||||
first=offset)
|
||||
search_path = search_string.format(query=urlencode({'q': query}), count=number_of_results, first=offset)
|
||||
|
||||
language = match_language(params['language'], supported_languages, language_aliases).lower()
|
||||
|
||||
params['cookies']['SRCHHPGUSR'] = \
|
||||
'ADLT=' + safesearch_types.get(params['safesearch'], 'DEMOTE')
|
||||
params['cookies']['SRCHHPGUSR'] = 'ADLT=' + safesearch_types.get(params['safesearch'], 'DEMOTE')
|
||||
|
||||
params['cookies']['_EDGE_S'] = 'mkt=' + language +\
|
||||
'&ui=' + language + '&F=1'
|
||||
params['cookies']['_EDGE_S'] = 'mkt=' + language + '&ui=' + language + '&F=1'
|
||||
|
||||
params['url'] = base_url + search_path
|
||||
if params['time_range'] in time_range_dict:
|
||||
|
|
@ -92,14 +89,18 @@ def response(resp):
|
|||
# strip 'Unicode private use area' highlighting, they render to Tux
|
||||
# the Linux penguin and a standing diamond on my machine...
|
||||
title = m.get('t', '').replace('\ue000', '').replace('\ue001', '')
|
||||
results.append({'template': 'images.html',
|
||||
results.append(
|
||||
{
|
||||
'template': 'images.html',
|
||||
'url': m['purl'],
|
||||
'thumbnail_src': m['turl'],
|
||||
'img_src': m['murl'],
|
||||
'content': '',
|
||||
'title': title,
|
||||
'source': source,
|
||||
'img_format': img_format})
|
||||
'img_format': img_format,
|
||||
}
|
||||
)
|
||||
except:
|
||||
continue
|
||||
|
||||
|
|
|
|||
|
|
@ -13,10 +13,7 @@ from datetime import datetime
|
|||
from dateutil import parser
|
||||
from lxml import etree
|
||||
from lxml.etree import XPath
|
||||
from searx.utils import (
|
||||
match_language,
|
||||
eval_xpath_getindex
|
||||
)
|
||||
from searx.utils import match_language, eval_xpath_getindex
|
||||
from searx.engines.bing import ( # pylint: disable=unused-import
|
||||
language_aliases,
|
||||
_fetch_supported_languages,
|
||||
|
|
@ -42,11 +39,8 @@ time_range_support = True
|
|||
base_url = 'https://www.bing.com/'
|
||||
search_string = 'news/search?{query}&first={offset}&format=RSS'
|
||||
search_string_with_time = 'news/search?{query}&first={offset}&qft=interval%3d"{interval}"&format=RSS'
|
||||
time_range_dict = {
|
||||
'day': '7',
|
||||
'week': '8',
|
||||
'month': '9'
|
||||
}
|
||||
time_range_dict = {'day': '7', 'week': '8', 'month': '9'}
|
||||
|
||||
|
||||
def url_cleanup(url_string):
|
||||
"""remove click"""
|
||||
|
|
@ -57,6 +51,7 @@ def url_cleanup(url_string):
|
|||
url_string = query.get('url', None)
|
||||
return url_string
|
||||
|
||||
|
||||
def image_url_cleanup(url_string):
|
||||
"""replace the http://*bing.com/th?id=... by https://www.bing.com/th?id=..."""
|
||||
|
||||
|
|
@ -66,27 +61,33 @@ def image_url_cleanup(url_string):
|
|||
url_string = "https://www.bing.com/th?id=" + quote(query.get('id'))
|
||||
return url_string
|
||||
|
||||
|
||||
def _get_url(query, language, offset, time_range):
|
||||
if time_range in time_range_dict:
|
||||
search_path = search_string_with_time.format(
|
||||
# fmt: off
|
||||
query = urlencode({
|
||||
'q': query,
|
||||
'setmkt': language
|
||||
}),
|
||||
offset = offset,
|
||||
interval = time_range_dict[time_range]
|
||||
# fmt: on
|
||||
)
|
||||
else:
|
||||
# e.g. setmkt=de-de&setlang=de
|
||||
search_path = search_string.format(
|
||||
# fmt: off
|
||||
query = urlencode({
|
||||
'q': query,
|
||||
'setmkt': language
|
||||
}),
|
||||
offset = offset
|
||||
# fmt: on
|
||||
)
|
||||
return base_url + search_path
|
||||
|
||||
|
||||
def request(query, params):
|
||||
|
||||
if params['time_range'] and params['time_range'] not in time_range_dict:
|
||||
|
|
@ -101,6 +102,7 @@ def request(query, params):
|
|||
|
||||
return params
|
||||
|
||||
|
||||
def response(resp):
|
||||
|
||||
results = []
|
||||
|
|
@ -123,26 +125,16 @@ def response(resp):
|
|||
publishedDate = datetime.now()
|
||||
|
||||
# thumbnail
|
||||
thumbnail = eval_xpath_getindex(
|
||||
item, XPath('./News:Image/text()', namespaces=namespaces), 0, default=None)
|
||||
thumbnail = eval_xpath_getindex(item, XPath('./News:Image/text()', namespaces=namespaces), 0, default=None)
|
||||
if thumbnail is not None:
|
||||
thumbnail = image_url_cleanup(thumbnail)
|
||||
|
||||
# append result
|
||||
if thumbnail is not None:
|
||||
results.append({
|
||||
'url': url,
|
||||
'title': title,
|
||||
'publishedDate': publishedDate,
|
||||
'content': content,
|
||||
'img_src': thumbnail
|
||||
})
|
||||
results.append(
|
||||
{'url': url, 'title': title, 'publishedDate': publishedDate, 'content': content, 'img_src': thumbnail}
|
||||
)
|
||||
else:
|
||||
results.append({
|
||||
'url': url,
|
||||
'title': title,
|
||||
'publishedDate': publishedDate,
|
||||
'content': content
|
||||
})
|
||||
results.append({'url': url, 'title': title, 'publishedDate': publishedDate, 'content': content})
|
||||
|
||||
return results
|
||||
|
|
|
|||
|
|
@ -6,12 +6,15 @@
|
|||
from json import loads
|
||||
from lxml import html
|
||||
from urllib.parse import urlencode
|
||||
|
||||
from searx.utils import match_language
|
||||
|
||||
from searx.engines.bing import language_aliases
|
||||
from searx.engines.bing import _fetch_supported_languages, supported_languages_url # NOQA # pylint: disable=unused-import
|
||||
|
||||
# about
|
||||
from searx.engines.bing import ( # pylint: disable=unused-import
|
||||
_fetch_supported_languages,
|
||||
supported_languages_url,
|
||||
)
|
||||
|
||||
about = {
|
||||
"website": 'https://www.bing.com/videos',
|
||||
"wikidata_id": 'Q4914152',
|
||||
|
|
@ -28,36 +31,31 @@ time_range_support = True
|
|||
number_of_results = 28
|
||||
|
||||
base_url = 'https://www.bing.com/'
|
||||
search_string = 'videos/search'\
|
||||
'?{query}'\
|
||||
'&count={count}'\
|
||||
'&first={first}'\
|
||||
'&scope=video'\
|
||||
search_string = (
|
||||
# fmt: off
|
||||
'videos/search'
|
||||
'?{query}'
|
||||
'&count={count}'
|
||||
'&first={first}'
|
||||
'&scope=video'
|
||||
'&FORM=QBLH'
|
||||
# fmt: on
|
||||
)
|
||||
time_range_string = '&qft=+filterui:videoage-lt{interval}'
|
||||
time_range_dict = {'day': '1440',
|
||||
'week': '10080',
|
||||
'month': '43200',
|
||||
'year': '525600'}
|
||||
time_range_dict = {'day': '1440', 'week': '10080', 'month': '43200', 'year': '525600'}
|
||||
|
||||
# safesearch definitions
|
||||
safesearch_types = {2: 'STRICT',
|
||||
1: 'DEMOTE',
|
||||
0: 'OFF'}
|
||||
safesearch_types = {2: 'STRICT', 1: 'DEMOTE', 0: 'OFF'}
|
||||
|
||||
|
||||
# do search-request
|
||||
def request(query, params):
|
||||
offset = ((params['pageno'] - 1) * number_of_results) + 1
|
||||
|
||||
search_path = search_string.format(
|
||||
query=urlencode({'q': query}),
|
||||
count=number_of_results,
|
||||
first=offset)
|
||||
search_path = search_string.format(query=urlencode({'q': query}), count=number_of_results, first=offset)
|
||||
|
||||
# safesearch cookie
|
||||
params['cookies']['SRCHHPGUSR'] = \
|
||||
'ADLT=' + safesearch_types.get(params['safesearch'], 'DEMOTE')
|
||||
params['cookies']['SRCHHPGUSR'] = 'ADLT=' + safesearch_types.get(params['safesearch'], 'DEMOTE')
|
||||
|
||||
# language cookie
|
||||
language = match_language(params['language'], supported_languages, language_aliases).lower()
|
||||
|
|
@ -89,11 +87,15 @@ def response(resp):
|
|||
info = ' - '.join(result.xpath('.//div[@class="mc_vtvc_meta_block"]//span/text()')).strip()
|
||||
content = '{0} - {1}'.format(metadata['du'], info)
|
||||
thumbnail = '{0}th?id={1}'.format(base_url, metadata['thid'])
|
||||
results.append({'url': metadata['murl'],
|
||||
results.append(
|
||||
{
|
||||
'url': metadata['murl'],
|
||||
'thumbnail': thumbnail,
|
||||
'title': metadata.get('vt', ''),
|
||||
'content': content,
|
||||
'template': 'videos.html'})
|
||||
'template': 'videos.html',
|
||||
}
|
||||
)
|
||||
|
||||
except:
|
||||
continue
|
||||
|
|
|
|||
|
|
@ -11,10 +11,7 @@ from searx.utils import extract_text, get_torrent_size
|
|||
about = {
|
||||
"website": 'https://btdig.com',
|
||||
"wikidata_id": 'Q4836698',
|
||||
"official_api_documentation": {
|
||||
'url': 'https://btdig.com/contacts',
|
||||
'comment': 'on demand'
|
||||
},
|
||||
"official_api_documentation": {'url': 'https://btdig.com/contacts', 'comment': 'on demand'},
|
||||
"use_official_api": False,
|
||||
"require_api_key": False,
|
||||
"results": 'HTML',
|
||||
|
|
@ -31,8 +28,7 @@ search_url = url + '/search?q={search_term}&p={pageno}'
|
|||
|
||||
# do search-request
|
||||
def request(query, params):
|
||||
params['url'] = search_url.format(search_term=quote(query),
|
||||
pageno=params['pageno'] - 1)
|
||||
params['url'] = search_url.format(search_term=quote(query), pageno=params['pageno'] - 1)
|
||||
|
||||
return params
|
||||
|
||||
|
|
@ -77,13 +73,17 @@ def response(resp):
|
|||
magnetlink = result.xpath('.//div[@class="torrent_magnet"]//a')[0].attrib['href']
|
||||
|
||||
# append result
|
||||
results.append({'url': href,
|
||||
results.append(
|
||||
{
|
||||
'url': href,
|
||||
'title': title,
|
||||
'content': content,
|
||||
'filesize': filesize,
|
||||
'files': files,
|
||||
'magnetlink': magnetlink,
|
||||
'template': 'torrent.html'})
|
||||
'template': 'torrent.html',
|
||||
}
|
||||
)
|
||||
|
||||
# return results sorted by seeder
|
||||
return results
|
||||
|
|
|
|||
|
|
@ -29,10 +29,7 @@ search_string = '&page={page}&page_size={nb_per_page}&format=json&{query}'
|
|||
|
||||
def request(query, params):
|
||||
|
||||
search_path = search_string.format(
|
||||
query=urlencode({'q': query}),
|
||||
nb_per_page=nb_per_page,
|
||||
page=params['pageno'])
|
||||
search_path = search_string.format(query=urlencode({'q': query}), nb_per_page=nb_per_page, page=params['pageno'])
|
||||
|
||||
params['url'] = base_url + search_path
|
||||
|
||||
|
|
@ -45,9 +42,13 @@ def response(resp):
|
|||
json_data = loads(resp.text)
|
||||
|
||||
for result in json_data['results']:
|
||||
results.append({'url': result['foreign_landing_url'],
|
||||
results.append(
|
||||
{
|
||||
'url': result['foreign_landing_url'],
|
||||
'title': result['title'],
|
||||
'img_src': result['url'],
|
||||
'template': 'images.html'})
|
||||
'template': 'images.html',
|
||||
}
|
||||
)
|
||||
|
||||
return results
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ api_key = 'unset'
|
|||
base_url = 'https://core.ac.uk:443/api-v2/search/'
|
||||
search_string = '{query}?page={page}&pageSize={nb_per_page}&apiKey={apikey}'
|
||||
|
||||
|
||||
def request(query, params):
|
||||
|
||||
if api_key == 'unset':
|
||||
|
|
@ -44,6 +45,7 @@ def request(query, params):
|
|||
logger.debug("query_url --> %s", params['url'])
|
||||
return params
|
||||
|
||||
|
||||
def response(resp):
|
||||
results = []
|
||||
json_data = loads(resp.text)
|
||||
|
|
@ -66,12 +68,14 @@ def response(resp):
|
|||
metadata.append(source['doi'])
|
||||
metadata = ' / '.join(metadata)
|
||||
|
||||
results.append({
|
||||
results.append(
|
||||
{
|
||||
'url': source['urls'][0].replace('http://', 'https://', 1),
|
||||
'title': source['title'],
|
||||
'content': source['description'],
|
||||
'publishedDate': date,
|
||||
'metadata': metadata,
|
||||
})
|
||||
}
|
||||
)
|
||||
|
||||
return results
|
||||
|
|
|
|||
|
|
@ -47,7 +47,8 @@ def response(resp):
|
|||
)
|
||||
|
||||
url = 'https://duckduckgo.com/js/spice/currency/1/{0}/{1}'.format(
|
||||
resp.search_params['from'].upper(), resp.search_params['to'])
|
||||
resp.search_params['from'].upper(), resp.search_params['to']
|
||||
)
|
||||
|
||||
results.append({'answer': answer, 'url': url})
|
||||
|
||||
|
|
|
|||
|
|
@ -25,8 +25,10 @@ paging = True
|
|||
# search-url
|
||||
# see http://www.dailymotion.com/doc/api/obj-video.html
|
||||
search_url = 'https://api.dailymotion.com/videos?fields=created_time,title,description,duration,url,thumbnail_360_url,id&sort=relevance&limit=5&page={pageno}&{query}' # noqa
|
||||
embedded_url = '<iframe frameborder="0" width="540" height="304" ' +\
|
||||
'data-src="https://www.dailymotion.com/embed/video/{videoid}" allowfullscreen></iframe>'
|
||||
embedded_url = (
|
||||
'<iframe frameborder="0" width="540" height="304" '
|
||||
+ 'data-src="https://www.dailymotion.com/embed/video/{videoid}" allowfullscreen></iframe>'
|
||||
)
|
||||
|
||||
supported_languages_url = 'https://api.dailymotion.com/languages'
|
||||
|
||||
|
|
@ -39,8 +41,8 @@ def request(query, params):
|
|||
locale = match_language(params['language'], supported_languages)
|
||||
|
||||
params['url'] = search_url.format(
|
||||
query=urlencode({'search': query, 'localization': locale}),
|
||||
pageno=params['pageno'])
|
||||
query=urlencode({'search': query, 'localization': locale}), pageno=params['pageno']
|
||||
)
|
||||
|
||||
return params
|
||||
|
||||
|
|
@ -67,13 +69,17 @@ def response(resp):
|
|||
# http to https
|
||||
thumbnail = thumbnail.replace("http://", "https://")
|
||||
|
||||
results.append({'template': 'videos.html',
|
||||
results.append(
|
||||
{
|
||||
'template': 'videos.html',
|
||||
'url': url,
|
||||
'title': title,
|
||||
'content': content,
|
||||
'publishedDate': publishedDate,
|
||||
'embedded': embedded,
|
||||
'thumbnail': thumbnail})
|
||||
'thumbnail': thumbnail,
|
||||
}
|
||||
)
|
||||
|
||||
# return results
|
||||
return results
|
||||
|
|
|
|||
|
|
@ -24,9 +24,11 @@ paging = True
|
|||
url = 'https://api.deezer.com/'
|
||||
search_url = url + 'search?{query}&index={offset}'
|
||||
|
||||
embedded_url = '<iframe scrolling="no" frameborder="0" allowTransparency="true" ' +\
|
||||
'data-src="https://www.deezer.com/plugins/player?type=tracks&id={audioid}" ' +\
|
||||
'width="540" height="80"></iframe>'
|
||||
embedded_url = (
|
||||
'<iframe scrolling="no" frameborder="0" allowTransparency="true" '
|
||||
+ 'data-src="https://www.deezer.com/plugins/player?type=tracks&id={audioid}" '
|
||||
+ 'width="540" height="80"></iframe>'
|
||||
)
|
||||
|
||||
|
||||
# do search-request
|
||||
|
|
@ -53,18 +55,12 @@ def response(resp):
|
|||
if url.startswith('http://'):
|
||||
url = 'https' + url[4:]
|
||||
|
||||
content = '{} - {} - {}'.format(
|
||||
result['artist']['name'],
|
||||
result['album']['title'],
|
||||
result['title'])
|
||||
content = '{} - {} - {}'.format(result['artist']['name'], result['album']['title'], result['title'])
|
||||
|
||||
embedded = embedded_url.format(audioid=result['id'])
|
||||
|
||||
# append result
|
||||
results.append({'url': url,
|
||||
'title': title,
|
||||
'embedded': embedded,
|
||||
'content': content})
|
||||
results.append({'url': url, 'title': title, 'embedded': embedded, 'content': content})
|
||||
|
||||
# return results
|
||||
return results
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ about = {
|
|||
# if there is a need for globals, use a leading underline
|
||||
_my_offline_engine = None
|
||||
|
||||
|
||||
def init(engine_settings=None):
|
||||
"""Initialization of the (offline) engine. The origin of this demo engine is a
|
||||
simple json string which is loaded in this example while the engine is
|
||||
|
|
@ -44,11 +45,10 @@ def init(engine_settings=None):
|
|||
', {"value":"first item"}'
|
||||
', {"value":"second item"}'
|
||||
', {"value":"third item"}'
|
||||
']'
|
||||
|
||||
% engine_settings.get('name')
|
||||
']' % engine_settings.get('name')
|
||||
)
|
||||
|
||||
|
||||
def search(query, request_params):
|
||||
"""Query (offline) engine and return results. Assemble the list of results from
|
||||
your local engine. In this demo engine we ignore the 'query' term, usual
|
||||
|
|
|
|||
|
|
@ -43,6 +43,7 @@ about = {
|
|||
# if there is a need for globals, use a leading underline
|
||||
_my_online_engine = None
|
||||
|
||||
|
||||
def init(engine_settings):
|
||||
"""Initialization of the (online) engine. If no initialization is needed, drop
|
||||
this init function.
|
||||
|
|
@ -51,20 +52,24 @@ def init(engine_settings):
|
|||
global _my_online_engine # pylint: disable=global-statement
|
||||
_my_online_engine = engine_settings.get('name')
|
||||
|
||||
|
||||
def request(query, params):
|
||||
"""Build up the ``params`` for the online request. In this example we build a
|
||||
URL to fetch images from `artic.edu <https://artic.edu>`__
|
||||
|
||||
"""
|
||||
args = urlencode({
|
||||
args = urlencode(
|
||||
{
|
||||
'q': query,
|
||||
'page': params['pageno'],
|
||||
'fields': 'id,title,artist_display,medium_display,image_id,date_display,dimensions,artist_titles',
|
||||
'limit': page_size,
|
||||
})
|
||||
}
|
||||
)
|
||||
params['url'] = search_api + args
|
||||
return params
|
||||
|
||||
|
||||
def response(resp):
|
||||
"""Parse out the result items from the response. In this example we parse the
|
||||
response from `api.artic.edu <https://artic.edu>`__ and filter out all
|
||||
|
|
@ -79,14 +84,16 @@ def response(resp):
|
|||
if not result['image_id']:
|
||||
continue
|
||||
|
||||
results.append({
|
||||
results.append(
|
||||
{
|
||||
'url': 'https://artic.edu/artworks/%(id)s' % result,
|
||||
'title': result['title'] + " (%(date_display)s) // %(artist_display)s" % result,
|
||||
'content': result['medium_display'],
|
||||
'author': ', '.join(result['artist_titles']),
|
||||
'img_src': image_api + '/%(image_id)s/full/843,/0/default.jpg' % result,
|
||||
'img_format': result['dimensions'],
|
||||
'template': 'images.html'
|
||||
})
|
||||
'template': 'images.html',
|
||||
}
|
||||
)
|
||||
|
||||
return results
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@ time_range_dict = {
|
|||
# search-url
|
||||
base_url = 'https://www.deviantart.com'
|
||||
|
||||
|
||||
def request(query, params):
|
||||
|
||||
# https://www.deviantart.com/search/deviations?page=5&q=foo
|
||||
|
|
@ -47,6 +48,7 @@ def request(query, params):
|
|||
|
||||
return params
|
||||
|
||||
|
||||
def response(resp):
|
||||
|
||||
results = []
|
||||
|
|
@ -67,11 +69,13 @@ def response(resp):
|
|||
continue
|
||||
img_tag = img_tag[0]
|
||||
|
||||
results.append({
|
||||
results.append(
|
||||
{
|
||||
'template': 'images.html',
|
||||
'url': a_tag.attrib.get('href'),
|
||||
'img_src': img_tag.attrib.get('src'),
|
||||
'title': img_tag.attrib.get('alt'),
|
||||
})
|
||||
}
|
||||
)
|
||||
|
||||
return results
|
||||
|
|
|
|||
|
|
@ -27,9 +27,7 @@ https_support = True
|
|||
|
||||
|
||||
def request(query, params):
|
||||
params['url'] = url.format(from_lang=params['from_lang'][2],
|
||||
to_lang=params['to_lang'][2],
|
||||
query=params['query'])
|
||||
params['url'] = url.format(from_lang=params['from_lang'][2], to_lang=params['to_lang'][2], query=params['query'])
|
||||
|
||||
return params
|
||||
|
||||
|
|
@ -51,10 +49,12 @@ def response(resp):
|
|||
if t.strip():
|
||||
to_results.append(to_result.text_content())
|
||||
|
||||
results.append({
|
||||
results.append(
|
||||
{
|
||||
'url': urljoin(str(resp.url), '?%d' % k),
|
||||
'title': from_result.text_content(),
|
||||
'content': '; '.join(to_results)
|
||||
})
|
||||
'content': '; '.join(to_results),
|
||||
}
|
||||
)
|
||||
|
||||
return results
|
||||
|
|
|
|||
|
|
@ -48,13 +48,17 @@ def response(resp):
|
|||
filesize = get_torrent_size(files_data[FILESIZE], files_data[FILESIZE_MULTIPLIER])
|
||||
magnetlink = result.xpath('.//div[@class="tail"]//a[@class="title"]/@href')[0]
|
||||
|
||||
results.append({'url': url,
|
||||
results.append(
|
||||
{
|
||||
'url': url,
|
||||
'title': title,
|
||||
'content': content,
|
||||
'filesize': filesize,
|
||||
'magnetlink': magnetlink,
|
||||
'seed': 'N/A',
|
||||
'leech': 'N/A',
|
||||
'template': 'torrent.html'})
|
||||
'template': 'torrent.html',
|
||||
}
|
||||
)
|
||||
|
||||
return results
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ paging = True
|
|||
base_url = "https://hub.docker.com/"
|
||||
search_url = base_url + "api/content/v1/products/search?{query}&type=image&page_size=25"
|
||||
|
||||
|
||||
def request(query, params):
|
||||
|
||||
params['url'] = search_url.format(query=urlencode(dict(q=query, page=params["pageno"])))
|
||||
|
|
@ -30,6 +31,7 @@ def request(query, params):
|
|||
|
||||
return params
|
||||
|
||||
|
||||
def response(resp):
|
||||
'''post-response callback
|
||||
resp: requests response object
|
||||
|
|
@ -53,12 +55,8 @@ def response(resp):
|
|||
result["url"] = base_url + "r/" + item.get('slug', "")
|
||||
result["title"] = item.get("name")
|
||||
result["content"] = item.get("short_description")
|
||||
result["publishedDate"] = parser.parse(
|
||||
item.get("updated_at") or item.get("created_at")
|
||||
)
|
||||
result["thumbnail"] = (
|
||||
item["logo_url"].get("large") or item["logo_url"].get("small")
|
||||
)
|
||||
result["publishedDate"] = parser.parse(item.get("updated_at") or item.get("created_at"))
|
||||
result["thumbnail"] = item["logo_url"].get("large") or item["logo_url"].get("small")
|
||||
results.append(result)
|
||||
|
||||
return results
|
||||
|
|
|
|||
|
|
@ -25,17 +25,20 @@ number_of_results = 5
|
|||
# search-url
|
||||
# Doku is OpenSearch compatible
|
||||
base_url = 'http://localhost:8090'
|
||||
search_url = '/?do=search'\
|
||||
search_url = (
|
||||
# fmt: off
|
||||
'/?do=search'
|
||||
'&{query}'
|
||||
# TODO '&startRecord={offset}'\
|
||||
# TODO '&maximumRecords={limit}'\
|
||||
# fmt: on
|
||||
)
|
||||
# TODO '&startRecord={offset}'
|
||||
# TODO '&maximumRecords={limit}'
|
||||
|
||||
|
||||
# do search-request
|
||||
def request(query, params):
|
||||
|
||||
params['url'] = base_url +\
|
||||
search_url.format(query=urlencode({'id': query}))
|
||||
params['url'] = base_url + search_url.format(query=urlencode({'id': query}))
|
||||
|
||||
return params
|
||||
|
||||
|
|
@ -60,9 +63,7 @@ def response(resp):
|
|||
title = extract_text(eval_xpath(r, './/a[@class="wikilink1"]/@title'))
|
||||
|
||||
# append result
|
||||
results.append({'title': title,
|
||||
'content': "",
|
||||
'url': base_url + res_url})
|
||||
results.append({'title': title, 'content': "", 'url': base_url + res_url})
|
||||
|
||||
# Search results
|
||||
for r in eval_xpath(doc, '//dl[@class="search_results"]/*'):
|
||||
|
|
@ -74,9 +75,7 @@ def response(resp):
|
|||
content = extract_text(eval_xpath(r, '.'))
|
||||
|
||||
# append result
|
||||
results.append({'title': title,
|
||||
'content': content,
|
||||
'url': base_url + res_url})
|
||||
results.append({'title': title, 'content': content, 'url': base_url + res_url})
|
||||
except:
|
||||
continue
|
||||
|
||||
|
|
|
|||
|
|
@ -39,15 +39,10 @@ language_aliases = {
|
|||
'ko': 'kr-KR',
|
||||
'sl-SI': 'sl-SL',
|
||||
'zh-TW': 'tzh-TW',
|
||||
'zh-HK': 'tzh-HK'
|
||||
'zh-HK': 'tzh-HK',
|
||||
}
|
||||
|
||||
time_range_dict = {
|
||||
'day': 'd',
|
||||
'week': 'w',
|
||||
'month': 'm',
|
||||
'year': 'y'
|
||||
}
|
||||
time_range_dict = {'day': 'd', 'week': 'w', 'month': 'm', 'year': 'y'}
|
||||
|
||||
# search-url
|
||||
url = 'https://lite.duckduckgo.com/lite'
|
||||
|
|
@ -118,6 +113,7 @@ def request(query, params):
|
|||
logger.debug("param cookies: %s", params['cookies'])
|
||||
return params
|
||||
|
||||
|
||||
# get response from search-request
|
||||
def response(resp):
|
||||
|
||||
|
|
@ -163,14 +159,17 @@ def response(resp):
|
|||
if td_content is None:
|
||||
continue
|
||||
|
||||
results.append({
|
||||
results.append(
|
||||
{
|
||||
'title': a_tag.text_content(),
|
||||
'content': extract_text(td_content),
|
||||
'url': a_tag.get('href'),
|
||||
})
|
||||
}
|
||||
)
|
||||
|
||||
return results
|
||||
|
||||
|
||||
# get supported languages from their site
|
||||
def _fetch_supported_languages(resp):
|
||||
|
||||
|
|
|
|||
|
|
@ -10,7 +10,10 @@ from lxml import html
|
|||
|
||||
from searx.data import WIKIDATA_UNITS
|
||||
from searx.engines.duckduckgo import language_aliases
|
||||
from searx.engines.duckduckgo import _fetch_supported_languages, supported_languages_url # NOQA # pylint: disable=unused-import
|
||||
from searx.engines.duckduckgo import ( # pylint: disable=unused-import
|
||||
_fetch_supported_languages,
|
||||
supported_languages_url,
|
||||
)
|
||||
from searx.utils import extract_text, html_to_text, match_language, get_string_replaces_function
|
||||
from searx.external_urls import get_external_url, get_earth_coordinates_url, area_to_osm_zoom
|
||||
|
||||
|
|
@ -24,13 +27,9 @@ about = {
|
|||
"results": 'JSON',
|
||||
}
|
||||
|
||||
URL = 'https://api.duckduckgo.com/'\
|
||||
+ '?{query}&format=json&pretty=0&no_redirect=1&d=1'
|
||||
URL = 'https://api.duckduckgo.com/' + '?{query}&format=json&pretty=0&no_redirect=1&d=1'
|
||||
|
||||
WIKIDATA_PREFIX = [
|
||||
'http://www.wikidata.org/entity/',
|
||||
'https://www.wikidata.org/entity/'
|
||||
]
|
||||
WIKIDATA_PREFIX = ['http://www.wikidata.org/entity/', 'https://www.wikidata.org/entity/']
|
||||
|
||||
replace_http_by_https = get_string_replaces_function({'http:': 'https:'})
|
||||
|
||||
|
|
@ -61,11 +60,7 @@ def result_to_text(text, htmlResult):
|
|||
|
||||
def request(query, params):
|
||||
params['url'] = URL.format(query=urlencode({'q': query}))
|
||||
language = match_language(
|
||||
params['language'],
|
||||
supported_languages,
|
||||
language_aliases
|
||||
)
|
||||
language = match_language(params['language'], supported_languages, language_aliases)
|
||||
language = language.split('-')[0]
|
||||
params['headers']['Accept-Language'] = language
|
||||
return params
|
||||
|
|
@ -127,23 +122,14 @@ def response(resp):
|
|||
firstURL = ddg_result.get('FirstURL')
|
||||
text = ddg_result.get('Text')
|
||||
if not is_broken_text(text):
|
||||
suggestion = result_to_text(
|
||||
text,
|
||||
ddg_result.get('Result')
|
||||
)
|
||||
suggestion = result_to_text(text, ddg_result.get('Result'))
|
||||
if suggestion != heading and suggestion is not None:
|
||||
results.append({'suggestion': suggestion})
|
||||
elif 'Topics' in ddg_result:
|
||||
suggestions = []
|
||||
relatedTopics.append({
|
||||
'name': ddg_result.get('Name', ''),
|
||||
'suggestions': suggestions
|
||||
})
|
||||
relatedTopics.append({'name': ddg_result.get('Name', ''), 'suggestions': suggestions})
|
||||
for topic_result in ddg_result.get('Topics', []):
|
||||
suggestion = result_to_text(
|
||||
topic_result.get('Text'),
|
||||
topic_result.get('Result')
|
||||
)
|
||||
suggestion = result_to_text(topic_result.get('Text'), topic_result.get('Result'))
|
||||
if suggestion != heading and suggestion is not None:
|
||||
suggestions.append(suggestion)
|
||||
|
||||
|
|
@ -152,25 +138,15 @@ def response(resp):
|
|||
if abstractURL != '':
|
||||
# add as result ? problem always in english
|
||||
infobox_id = abstractURL
|
||||
urls.append({
|
||||
'title': search_res.get('AbstractSource'),
|
||||
'url': abstractURL,
|
||||
'official': True
|
||||
})
|
||||
results.append({
|
||||
'url': abstractURL,
|
||||
'title': heading
|
||||
})
|
||||
urls.append({'title': search_res.get('AbstractSource'), 'url': abstractURL, 'official': True})
|
||||
results.append({'url': abstractURL, 'title': heading})
|
||||
|
||||
# definition
|
||||
definitionURL = search_res.get('DefinitionURL', '')
|
||||
if definitionURL != '':
|
||||
# add as result ? as answer ? problem always in english
|
||||
infobox_id = definitionURL
|
||||
urls.append({
|
||||
'title': search_res.get('DefinitionSource'),
|
||||
'url': definitionURL
|
||||
})
|
||||
urls.append({'title': search_res.get('DefinitionSource'), 'url': definitionURL})
|
||||
|
||||
# to merge with wikidata's infobox
|
||||
if infobox_id:
|
||||
|
|
@ -198,10 +174,7 @@ def response(resp):
|
|||
# * netflix_id
|
||||
external_url = get_external_url(data_type, data_value)
|
||||
if external_url is not None:
|
||||
urls.append({
|
||||
'title': data_label,
|
||||
'url': external_url
|
||||
})
|
||||
urls.append({'title': data_label, 'url': external_url})
|
||||
elif data_type in ['instance', 'wiki_maps_trigger', 'google_play_artist_id']:
|
||||
# ignore instance: Wikidata value from "Instance Of" (Qxxxx)
|
||||
# ignore wiki_maps_trigger: reference to a javascript
|
||||
|
|
@ -211,11 +184,7 @@ def response(resp):
|
|||
# There is already an URL for the website
|
||||
pass
|
||||
elif data_type == 'area':
|
||||
attributes.append({
|
||||
'label': data_label,
|
||||
'value': area_to_str(data_value),
|
||||
'entity': 'P2046'
|
||||
})
|
||||
attributes.append({'label': data_label, 'value': area_to_str(data_value), 'entity': 'P2046'})
|
||||
osm_zoom = area_to_osm_zoom(data_value.get('amount'))
|
||||
elif data_type == 'coordinates':
|
||||
if data_value.get('globe') == 'http://www.wikidata.org/entity/Q2':
|
||||
|
|
@ -224,16 +193,9 @@ def response(resp):
|
|||
coordinates = info
|
||||
else:
|
||||
# coordinate NOT on Earth
|
||||
attributes.append({
|
||||
'label': data_label,
|
||||
'value': data_value,
|
||||
'entity': 'P625'
|
||||
})
|
||||
attributes.append({'label': data_label, 'value': data_value, 'entity': 'P625'})
|
||||
elif data_type == 'string':
|
||||
attributes.append({
|
||||
'label': data_label,
|
||||
'value': data_value
|
||||
})
|
||||
attributes.append({'label': data_label, 'value': data_value})
|
||||
|
||||
if coordinates:
|
||||
data_label = coordinates.get('label')
|
||||
|
|
@ -241,31 +203,24 @@ def response(resp):
|
|||
latitude = data_value.get('latitude')
|
||||
longitude = data_value.get('longitude')
|
||||
url = get_earth_coordinates_url(latitude, longitude, osm_zoom)
|
||||
urls.append({
|
||||
'title': 'OpenStreetMap',
|
||||
'url': url,
|
||||
'entity': 'P625'
|
||||
})
|
||||
urls.append({'title': 'OpenStreetMap', 'url': url, 'entity': 'P625'})
|
||||
|
||||
if len(heading) > 0:
|
||||
# TODO get infobox.meta.value where .label='article_title' # pylint: disable=fixme
|
||||
if image is None and len(attributes) == 0 and len(urls) == 1 and\
|
||||
len(relatedTopics) == 0 and len(content) == 0:
|
||||
results.append({
|
||||
'url': urls[0]['url'],
|
||||
'title': heading,
|
||||
'content': content
|
||||
})
|
||||
if image is None and len(attributes) == 0 and len(urls) == 1 and len(relatedTopics) == 0 and len(content) == 0:
|
||||
results.append({'url': urls[0]['url'], 'title': heading, 'content': content})
|
||||
else:
|
||||
results.append({
|
||||
results.append(
|
||||
{
|
||||
'infobox': heading,
|
||||
'id': infobox_id,
|
||||
'content': content,
|
||||
'img_src': image,
|
||||
'attributes': attributes,
|
||||
'urls': urls,
|
||||
'relatedTopics': relatedTopics
|
||||
})
|
||||
'relatedTopics': relatedTopics,
|
||||
}
|
||||
)
|
||||
|
||||
return results
|
||||
|
||||
|
|
|
|||
|
|
@ -7,7 +7,10 @@ from json import loads
|
|||
from urllib.parse import urlencode
|
||||
from searx.exceptions import SearxEngineAPIException
|
||||
from searx.engines.duckduckgo import get_region_code
|
||||
from searx.engines.duckduckgo import _fetch_supported_languages, supported_languages_url # NOQA # pylint: disable=unused-import
|
||||
from searx.engines.duckduckgo import ( # pylint: disable=unused-import
|
||||
_fetch_supported_languages,
|
||||
supported_languages_url,
|
||||
)
|
||||
from searx.network import get
|
||||
|
||||
# about
|
||||
|
|
@ -61,10 +64,10 @@ def request(query, params):
|
|||
region_code = get_region_code(params['language'], lang_list=supported_languages)
|
||||
if region_code:
|
||||
params['url'] = images_url.format(
|
||||
query=urlencode({'q': query, 'l': region_code}), offset=offset, safesearch=safesearch, vqd=vqd)
|
||||
query=urlencode({'q': query, 'l': region_code}), offset=offset, safesearch=safesearch, vqd=vqd
|
||||
)
|
||||
else:
|
||||
params['url'] = images_url.format(
|
||||
query=urlencode({'q': query}), offset=offset, safesearch=safesearch, vqd=vqd)
|
||||
params['url'] = images_url.format(query=urlencode({'q': query}), offset=offset, safesearch=safesearch, vqd=vqd)
|
||||
|
||||
return params
|
||||
|
||||
|
|
@ -84,11 +87,15 @@ def response(resp):
|
|||
image = result['image']
|
||||
|
||||
# append result
|
||||
results.append({'template': 'images.html',
|
||||
results.append(
|
||||
{
|
||||
'template': 'images.html',
|
||||
'title': title,
|
||||
'content': '',
|
||||
'thumbnail_src': thumbnail,
|
||||
'img_src': image,
|
||||
'url': url})
|
||||
'url': url,
|
||||
}
|
||||
)
|
||||
|
||||
return results
|
||||
|
|
|
|||
|
|
@ -38,7 +38,7 @@ def request(query, params):
|
|||
pageno : 1 # number of the requested page
|
||||
'''
|
||||
|
||||
offset = (params['pageno'] - 1)
|
||||
offset = params['pageno'] - 1
|
||||
if offset == 0:
|
||||
search_url_fmt = base_url + 'suchen/dudenonline/{query}'
|
||||
params['url'] = search_url_fmt.format(query=quote(query))
|
||||
|
|
@ -58,9 +58,9 @@ def response(resp):
|
|||
|
||||
dom = html.fromstring(resp.text)
|
||||
|
||||
number_of_results_element =\
|
||||
eval_xpath_getindex(dom, '//a[@class="active" and contains(@href,"/suchen/dudenonline")]/span/text()',
|
||||
0, default=None)
|
||||
number_of_results_element = eval_xpath_getindex(
|
||||
dom, '//a[@class="active" and contains(@href,"/suchen/dudenonline")]/span/text()', 0, default=None
|
||||
)
|
||||
if number_of_results_element is not None:
|
||||
number_of_results_string = re.sub('[^0-9]', '', number_of_results_element)
|
||||
results.append({'number_of_results': int(number_of_results_string)})
|
||||
|
|
@ -71,8 +71,6 @@ def response(resp):
|
|||
title = eval_xpath(result, 'string(.//h2/a)').strip()
|
||||
content = extract_text(eval_xpath(result, './/p'))
|
||||
# append result
|
||||
results.append({'url': url,
|
||||
'title': title,
|
||||
'content': content})
|
||||
results.append({'url': url, 'title': title, 'content': content})
|
||||
|
||||
return results
|
||||
|
|
|
|||
|
|
@ -15,6 +15,8 @@ about = {
|
|||
|
||||
|
||||
def search(query, request_params):
|
||||
return [{
|
||||
return [
|
||||
{
|
||||
'result': 'this is what you get',
|
||||
}]
|
||||
}
|
||||
]
|
||||
|
|
|
|||
|
|
@ -58,7 +58,8 @@ def response(resp):
|
|||
if title == "":
|
||||
continue
|
||||
|
||||
results.append({
|
||||
results.append(
|
||||
{
|
||||
'url': url,
|
||||
'title': title,
|
||||
'content': content,
|
||||
|
|
@ -67,7 +68,7 @@ def response(resp):
|
|||
'source_country': source_country,
|
||||
'thumbnail': thumbnail,
|
||||
'template': 'products.html',
|
||||
|
||||
})
|
||||
}
|
||||
)
|
||||
|
||||
return results
|
||||
|
|
|
|||
|
|
@ -119,9 +119,7 @@ def response(resp):
|
|||
r['template'] = 'key-value.html'
|
||||
|
||||
if show_metadata:
|
||||
r['metadata'] = {'index': result['_index'],
|
||||
'id': result['_id'],
|
||||
'score': result['_score']}
|
||||
r['metadata'] = {'index': result['_index'], 'id': result['_id'], 'score': result['_score']}
|
||||
|
||||
results.append(r)
|
||||
|
||||
|
|
@ -133,12 +131,10 @@ _available_query_types = {
|
|||
# https://www.elastic.co/guide/en/elasticsearch/reference/current/full-text-queries.html
|
||||
'match': _match_query,
|
||||
'simple_query_string': _simple_query_string_query,
|
||||
|
||||
# Term-level queries
|
||||
# https://www.elastic.co/guide/en/elasticsearch/reference/current/term-level-queries.html
|
||||
'term': _term_query,
|
||||
'terms': _terms_query,
|
||||
|
||||
# Query JSON defined by the instance administrator.
|
||||
'custom': _custom_query,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -22,10 +22,14 @@ paging = False
|
|||
safesearch = True
|
||||
|
||||
base_url = 'https://www.etools.ch'
|
||||
search_path = '/searchAdvancedSubmit.do'\
|
||||
'?query={search_term}'\
|
||||
'&pageResults=20'\
|
||||
search_path = (
|
||||
# fmt: off
|
||||
'/searchAdvancedSubmit.do'
|
||||
'?query={search_term}'
|
||||
'&pageResults=20'
|
||||
'&safeSearch={safesearch}'
|
||||
# fmt: on
|
||||
)
|
||||
|
||||
|
||||
def request(query, params):
|
||||
|
|
@ -49,8 +53,6 @@ def response(resp):
|
|||
title = extract_text(eval_xpath(result, './a//text()'))
|
||||
content = extract_text(eval_xpath(result, './/div[@class="text"]//text()'))
|
||||
|
||||
results.append({'url': url,
|
||||
'title': title,
|
||||
'content': content})
|
||||
results.append({'url': url, 'title': title, 'content': content})
|
||||
|
||||
return results
|
||||
|
|
|
|||
|
|
@ -42,13 +42,13 @@ def response(resp):
|
|||
for app in dom.xpath('//a[@class="package-header"]'):
|
||||
app_url = app.xpath('./@href')[0]
|
||||
app_title = extract_text(app.xpath('./div/h4[@class="package-name"]/text()'))
|
||||
app_content = extract_text(app.xpath('./div/div/span[@class="package-summary"]')).strip() \
|
||||
+ ' - ' + extract_text(app.xpath('./div/div/span[@class="package-license"]')).strip()
|
||||
app_content = (
|
||||
extract_text(app.xpath('./div/div/span[@class="package-summary"]')).strip()
|
||||
+ ' - '
|
||||
+ extract_text(app.xpath('./div/div/span[@class="package-license"]')).strip()
|
||||
)
|
||||
app_img_src = app.xpath('./img[@class="package-icon"]/@src')[0]
|
||||
|
||||
results.append({'url': app_url,
|
||||
'title': app_title,
|
||||
'content': app_content,
|
||||
'img_src': app_img_src})
|
||||
results.append({'url': app_url, 'title': app_title, 'content': app_content, 'img_src': app_img_src})
|
||||
|
||||
return results
|
||||
|
|
|
|||
|
|
@ -25,10 +25,12 @@ paging = True
|
|||
api_key = None
|
||||
|
||||
|
||||
url = 'https://api.flickr.com/services/rest/?method=flickr.photos.search' +\
|
||||
'&api_key={api_key}&{text}&sort=relevance' +\
|
||||
'&extras=description%2C+owner_name%2C+url_o%2C+url_n%2C+url_z' +\
|
||||
'&per_page={nb_per_page}&format=json&nojsoncallback=1&page={page}'
|
||||
url = (
|
||||
'https://api.flickr.com/services/rest/?method=flickr.photos.search'
|
||||
+ '&api_key={api_key}&{text}&sort=relevance'
|
||||
+ '&extras=description%2C+owner_name%2C+url_o%2C+url_n%2C+url_z'
|
||||
+ '&per_page={nb_per_page}&format=json&nojsoncallback=1&page={page}'
|
||||
)
|
||||
photo_url = 'https://www.flickr.com/photos/{userid}/{photoid}'
|
||||
|
||||
paging = True
|
||||
|
|
@ -39,10 +41,9 @@ def build_flickr_url(user_id, photo_id):
|
|||
|
||||
|
||||
def request(query, params):
|
||||
params['url'] = url.format(text=urlencode({'text': query}),
|
||||
api_key=api_key,
|
||||
nb_per_page=nb_per_page,
|
||||
page=params['pageno'])
|
||||
params['url'] = url.format(
|
||||
text=urlencode({'text': query}), api_key=api_key, nb_per_page=nb_per_page, page=params['pageno']
|
||||
)
|
||||
return params
|
||||
|
||||
|
||||
|
|
@ -80,13 +81,17 @@ def response(resp):
|
|||
url = build_flickr_url(photo['owner'], photo['id'])
|
||||
|
||||
# append result
|
||||
results.append({'url': url,
|
||||
results.append(
|
||||
{
|
||||
'url': url,
|
||||
'title': photo['title'],
|
||||
'img_src': img_src,
|
||||
'thumbnail_src': thumbnail_src,
|
||||
'content': photo['description']['_content'],
|
||||
'author': photo['ownername'],
|
||||
'template': 'images.html'})
|
||||
'template': 'images.html',
|
||||
}
|
||||
)
|
||||
|
||||
# return results
|
||||
return results
|
||||
|
|
|
|||
|
|
@ -30,10 +30,12 @@ image_sizes = ('o', 'k', 'h', 'b', 'c', 'z', 'n', 'm', 't', 'q', 's')
|
|||
|
||||
paging = True
|
||||
time_range_support = True
|
||||
time_range_dict = {'day': 60 * 60 * 24,
|
||||
time_range_dict = {
|
||||
'day': 60 * 60 * 24,
|
||||
'week': 60 * 60 * 24 * 7,
|
||||
'month': 60 * 60 * 24 * 7 * 4,
|
||||
'year': 60 * 60 * 24 * 7 * 52}
|
||||
'year': 60 * 60 * 24 * 7 * 52,
|
||||
}
|
||||
|
||||
|
||||
def build_flickr_url(user_id, photo_id):
|
||||
|
|
@ -47,8 +49,9 @@ def _get_time_range_url(time_range):
|
|||
|
||||
|
||||
def request(query, params):
|
||||
params['url'] = (search_url.format(query=urlencode({'text': query}), page=params['pageno'])
|
||||
+ _get_time_range_url(params['time_range']))
|
||||
params['url'] = search_url.format(query=urlencode({'text': query}), page=params['pageno']) + _get_time_range_url(
|
||||
params['time_range']
|
||||
)
|
||||
return params
|
||||
|
||||
|
||||
|
|
@ -83,10 +86,9 @@ def response(resp):
|
|||
for image_size in image_sizes:
|
||||
if image_size in photo['sizes']:
|
||||
img_src = photo['sizes'][image_size]['url']
|
||||
img_format = 'jpg ' \
|
||||
+ str(photo['sizes'][image_size]['width']) \
|
||||
+ 'x' \
|
||||
+ str(photo['sizes'][image_size]['height'])
|
||||
img_format = (
|
||||
'jpg ' + str(photo['sizes'][image_size]['width']) + 'x' + str(photo['sizes'][image_size]['height'])
|
||||
)
|
||||
break
|
||||
|
||||
if not img_src:
|
||||
|
|
@ -113,7 +115,7 @@ def response(resp):
|
|||
'thumbnail_src': thumbnail_src,
|
||||
'source': source,
|
||||
'img_format': img_format,
|
||||
'template': 'images.html'
|
||||
'template': 'images.html',
|
||||
}
|
||||
result['author'] = author.encode(errors='ignore').decode()
|
||||
result['source'] = source.encode(errors='ignore').decode()
|
||||
|
|
|
|||
|
|
@ -35,9 +35,8 @@ content_xpath = './/div[@class="content"]//p'
|
|||
|
||||
# do search-request
|
||||
def request(query, params):
|
||||
offset = (params['pageno'] - 1)
|
||||
params['url'] = search_url.format(query=urlencode({'keys': query}),
|
||||
offset=offset)
|
||||
offset = params['pageno'] - 1
|
||||
params['url'] = search_url.format(query=urlencode({'keys': query}), offset=offset)
|
||||
|
||||
return params
|
||||
|
||||
|
|
@ -63,10 +62,7 @@ def response(resp):
|
|||
content = escape(extract_text(result.xpath(content_xpath)))
|
||||
|
||||
# append result
|
||||
results.append({'url': href,
|
||||
'title': title,
|
||||
'img_src': thumbnail,
|
||||
'content': content})
|
||||
results.append({'url': href, 'title': title, 'img_src': thumbnail, 'content': content})
|
||||
|
||||
# return results
|
||||
return results
|
||||
|
|
|
|||
|
|
@ -26,8 +26,7 @@ paging = True
|
|||
# search url
|
||||
url = "https://freesound.org/apiv2/"
|
||||
search_url = (
|
||||
url
|
||||
+ "search/text/?query={query}&page={page}&fields=name,url,download,created,description,type&token={api_key}"
|
||||
url + "search/text/?query={query}&page={page}&fields=name,url,download,created,description,type&token={api_key}"
|
||||
)
|
||||
|
||||
embedded_url = '<audio controls><source src="{uri}" type="audio/{ftype}"></audio>'
|
||||
|
|
|
|||
|
|
@ -10,10 +10,7 @@ from urllib.parse import urlencode
|
|||
about = {
|
||||
"website": 'https://frinkiac.com',
|
||||
"wikidata_id": 'Q24882614',
|
||||
"official_api_documentation": {
|
||||
'url': None,
|
||||
'comment': 'see https://github.com/MitchellAW/CompuGlobal'
|
||||
},
|
||||
"official_api_documentation": {'url': None, 'comment': 'see https://github.com/MitchellAW/CompuGlobal'},
|
||||
"use_official_api": False,
|
||||
"require_api_key": False,
|
||||
"results": 'JSON',
|
||||
|
|
@ -40,12 +37,15 @@ def response(resp):
|
|||
episode = result['Episode']
|
||||
timestamp = result['Timestamp']
|
||||
|
||||
results.append({'template': 'images.html',
|
||||
'url': RESULT_URL.format(base=BASE,
|
||||
query=urlencode({'p': 'caption', 'e': episode, 't': timestamp})),
|
||||
results.append(
|
||||
{
|
||||
'template': 'images.html',
|
||||
'url': RESULT_URL.format(base=BASE, query=urlencode({'p': 'caption', 'e': episode, 't': timestamp})),
|
||||
'title': episode,
|
||||
'content': '',
|
||||
'thumbnail_src': THUMB_URL.format(base=BASE, episode=episode, timestamp=timestamp),
|
||||
'img_src': IMAGE_URL.format(base=BASE, episode=episode, timestamp=timestamp)})
|
||||
'img_src': IMAGE_URL.format(base=BASE, episode=episode, timestamp=timestamp),
|
||||
}
|
||||
)
|
||||
|
||||
return results
|
||||
|
|
|
|||
|
|
@ -37,15 +37,12 @@ def locale_to_lang_code(locale):
|
|||
# wikis for some languages were moved off from the main site, we need to make
|
||||
# requests to correct URLs to be able to get results in those languages
|
||||
lang_urls = {
|
||||
'en': {
|
||||
'base': 'https://wiki.gentoo.org',
|
||||
'search': '/index.php?title=Special:Search&offset={offset}&{query}'
|
||||
},
|
||||
'en': {'base': 'https://wiki.gentoo.org', 'search': '/index.php?title=Special:Search&offset={offset}&{query}'},
|
||||
'others': {
|
||||
'base': 'https://wiki.gentoo.org',
|
||||
'search': '/index.php?title=Special:Search&offset={offset}&{query}\
|
||||
&profile=translation&languagefilter={language}'
|
||||
}
|
||||
&profile=translation&languagefilter={language}',
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -78,7 +75,7 @@ main_langs = {
|
|||
'sl': 'Slovenský',
|
||||
'th': 'ไทย',
|
||||
'uk': 'Українська',
|
||||
'zh': '简体中文'
|
||||
'zh': '简体中文',
|
||||
}
|
||||
supported_languages = dict(lang_urls, **main_langs)
|
||||
|
||||
|
|
@ -101,8 +98,7 @@ def request(query, params):
|
|||
urls = get_lang_urls(language)
|
||||
search_url = urls['base'] + urls['search']
|
||||
|
||||
params['url'] = search_url.format(query=query, offset=offset,
|
||||
language=language)
|
||||
params['url'] = search_url.format(query=query, offset=offset, language=language)
|
||||
|
||||
return params
|
||||
|
||||
|
|
@ -123,7 +119,6 @@ def response(resp):
|
|||
href = urljoin(base_url, link.attrib.get('href'))
|
||||
title = extract_text(link)
|
||||
|
||||
results.append({'url': href,
|
||||
'title': title})
|
||||
results.append({'url': href, 'title': title})
|
||||
|
||||
return results
|
||||
|
|
|
|||
|
|
@ -69,12 +69,7 @@ def fetch_extra_param(query_args, headers):
|
|||
|
||||
# do search-request
|
||||
def request(query, params): # pylint: disable=unused-argument
|
||||
query_args = dict(
|
||||
c = 'main'
|
||||
, q = query
|
||||
, dr = 1
|
||||
, showgoodimages = 0
|
||||
)
|
||||
query_args = dict(c='main', q=query, dr=1, showgoodimages=0)
|
||||
|
||||
if params['language'] and params['language'] != 'all':
|
||||
query_args['qlangcountry'] = params['language']
|
||||
|
|
@ -93,6 +88,7 @@ def request(query, params): # pylint: disable=unused-argument
|
|||
|
||||
return params
|
||||
|
||||
|
||||
# get response from search-request
|
||||
def response(resp):
|
||||
results = []
|
||||
|
|
@ -125,10 +121,6 @@ def response(resp):
|
|||
if len(subtitle) > 3 and subtitle != title:
|
||||
title += " - " + subtitle
|
||||
|
||||
results.append(dict(
|
||||
url = url
|
||||
, title = title
|
||||
, content = content
|
||||
))
|
||||
results.append(dict(url=url, title=title, content=content))
|
||||
|
||||
return results
|
||||
|
|
|
|||
|
|
@ -55,9 +55,7 @@ def response(resp):
|
|||
content = ''
|
||||
|
||||
# append result
|
||||
results.append({'url': url,
|
||||
'title': title,
|
||||
'content': content})
|
||||
results.append({'url': url, 'title': title, 'content': content})
|
||||
|
||||
# return results
|
||||
return results
|
||||
|
|
|
|||
|
|
@ -100,22 +100,13 @@ google_domains = {
|
|||
'UA': 'google.com.ua', # Ukraine
|
||||
'CN': 'google.com.hk', # There is no google.cn, we use .com.hk for zh-CN
|
||||
'HK': 'google.com.hk', # Hong Kong
|
||||
'TW': 'google.com.tw' # Taiwan
|
||||
'TW': 'google.com.tw', # Taiwan
|
||||
}
|
||||
|
||||
time_range_dict = {
|
||||
'day': 'd',
|
||||
'week': 'w',
|
||||
'month': 'm',
|
||||
'year': 'y'
|
||||
}
|
||||
time_range_dict = {'day': 'd', 'week': 'w', 'month': 'm', 'year': 'y'}
|
||||
|
||||
# Filter results. 0: None, 1: Moderate, 2: Strict
|
||||
filter_mapping = {
|
||||
0: 'off',
|
||||
1: 'medium',
|
||||
2: 'high'
|
||||
}
|
||||
filter_mapping = {0: 'off', 1: 'medium', 2: 'high'}
|
||||
|
||||
# specific xpath variables
|
||||
# ------------------------
|
||||
|
|
@ -140,6 +131,7 @@ content_xpath = './/div[@class="IsZvec"]'
|
|||
# from the links not the links itself.
|
||||
suggestion_xpath = '//div[contains(@class, "EIaa9b")]//a'
|
||||
|
||||
|
||||
def get_lang_info(params, lang_list, custom_aliases, supported_any_language):
|
||||
"""Composing various language properties for the google engines.
|
||||
|
||||
|
|
@ -250,15 +242,18 @@ def get_lang_info(params, lang_list, custom_aliases, supported_any_language):
|
|||
ret_val['params']['lr'] = "lang_" + lang_list.get(lang_country, language)
|
||||
|
||||
# Accept-Language: fr-CH, fr;q=0.8, en;q=0.6, *;q=0.5
|
||||
ret_val['headers']['Accept-Language'] = ','.join([
|
||||
ret_val['headers']['Accept-Language'] = ','.join(
|
||||
[
|
||||
lang_country,
|
||||
language + ';q=0.8,',
|
||||
'en;q=0.6',
|
||||
'*;q=0.5',
|
||||
])
|
||||
]
|
||||
)
|
||||
|
||||
return ret_val
|
||||
|
||||
|
||||
def detect_google_sorry(resp):
|
||||
if resp.url.host == 'sorry.google.com' or resp.url.path.startswith('/sorry'):
|
||||
raise SearxEngineCaptchaException()
|
||||
|
|
@ -269,9 +264,7 @@ def request(query, params):
|
|||
|
||||
offset = (params['pageno'] - 1) * 10
|
||||
|
||||
lang_info = get_lang_info(
|
||||
params, supported_languages, language_aliases, True
|
||||
)
|
||||
lang_info = get_lang_info(params, supported_languages, language_aliases, True)
|
||||
|
||||
additional_parameters = {}
|
||||
if use_mobile_ui:
|
||||
|
|
@ -281,7 +274,13 @@ def request(query, params):
|
|||
}
|
||||
|
||||
# https://www.google.de/search?q=corona&hl=de&lr=lang_de&start=0&tbs=qdr%3Ad&safe=medium
|
||||
query_url = 'https://' + lang_info['subdomain'] + '/search' + "?" + urlencode({
|
||||
query_url = (
|
||||
'https://'
|
||||
+ lang_info['subdomain']
|
||||
+ '/search'
|
||||
+ "?"
|
||||
+ urlencode(
|
||||
{
|
||||
'q': query,
|
||||
**lang_info['params'],
|
||||
'ie': "utf8",
|
||||
|
|
@ -289,7 +288,9 @@ def request(query, params):
|
|||
'start': offset,
|
||||
'filter': '0',
|
||||
**additional_parameters,
|
||||
})
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
if params['time_range'] in time_range_dict:
|
||||
query_url += '&' + urlencode({'tbs': 'qdr:' + time_range_dict[params['time_range']]})
|
||||
|
|
@ -301,9 +302,7 @@ def request(query, params):
|
|||
if use_mobile_ui:
|
||||
params['headers']['Accept'] = '*/*'
|
||||
else:
|
||||
params['headers']['Accept'] = (
|
||||
'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
|
||||
)
|
||||
params['headers']['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
|
||||
|
||||
return params
|
||||
|
||||
|
|
@ -355,11 +354,7 @@ def response(resp):
|
|||
if url is None:
|
||||
continue
|
||||
content = extract_text(eval_xpath_getindex(result, content_xpath, 0, default=None), allow_none=True)
|
||||
results.append({
|
||||
'url': url,
|
||||
'title': title,
|
||||
'content': content
|
||||
})
|
||||
results.append({'url': url, 'title': title, 'content': content})
|
||||
except Exception as e: # pylint: disable=broad-except
|
||||
logger.error(e, exc_info=True)
|
||||
# from lxml import etree
|
||||
|
|
|
|||
|
|
@ -30,10 +30,8 @@ from searx.engines.google import (
|
|||
)
|
||||
|
||||
# pylint: disable=unused-import
|
||||
from searx.engines.google import (
|
||||
supported_languages_url
|
||||
, _fetch_supported_languages
|
||||
)
|
||||
from searx.engines.google import supported_languages_url, _fetch_supported_languages
|
||||
|
||||
# pylint: enable=unused-import
|
||||
|
||||
# about
|
||||
|
|
@ -53,16 +51,11 @@ use_locale_domain = True
|
|||
time_range_support = True
|
||||
safesearch = True
|
||||
|
||||
filter_mapping = {
|
||||
0: 'images',
|
||||
1: 'active',
|
||||
2: 'active'
|
||||
}
|
||||
filter_mapping = {0: 'images', 1: 'active', 2: 'active'}
|
||||
|
||||
|
||||
def scrap_out_thumbs(dom):
|
||||
"""Scrap out thumbnail data from <script> tags.
|
||||
"""
|
||||
"""Scrap out thumbnail data from <script> tags."""
|
||||
ret_val = {}
|
||||
for script in eval_xpath(dom, '//script[contains(., "_setImgSrc(")]'):
|
||||
_script = script.text
|
||||
|
|
@ -76,8 +69,7 @@ def scrap_out_thumbs(dom):
|
|||
|
||||
|
||||
def scrap_img_by_id(script, data_id):
|
||||
"""Get full image URL by data-id in parent element
|
||||
"""
|
||||
"""Get full image URL by data-id in parent element"""
|
||||
img_url = ''
|
||||
_script = script.split('\n')
|
||||
for i, line in enumerate(_script):
|
||||
|
|
@ -91,20 +83,25 @@ def scrap_img_by_id(script, data_id):
|
|||
def request(query, params):
|
||||
"""Google-Video search request"""
|
||||
|
||||
lang_info = get_lang_info(
|
||||
params, supported_languages, language_aliases, False
|
||||
)
|
||||
logger.debug(
|
||||
"HTTP header Accept-Language --> %s", lang_info['headers']['Accept-Language'])
|
||||
lang_info = get_lang_info(params, supported_languages, language_aliases, False)
|
||||
logger.debug("HTTP header Accept-Language --> %s", lang_info['headers']['Accept-Language'])
|
||||
|
||||
query_url = 'https://' + lang_info['subdomain'] + '/search' + "?" + urlencode({
|
||||
query_url = (
|
||||
'https://'
|
||||
+ lang_info['subdomain']
|
||||
+ '/search'
|
||||
+ "?"
|
||||
+ urlencode(
|
||||
{
|
||||
'q': query,
|
||||
'tbm': "isch",
|
||||
**lang_info['params'],
|
||||
'ie': "utf8",
|
||||
'oe': "utf8",
|
||||
'num': 30,
|
||||
})
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
if params['time_range'] in time_range_dict:
|
||||
query_url += '&' + urlencode({'tbs': 'qdr:' + time_range_dict[params['time_range']]})
|
||||
|
|
@ -113,9 +110,7 @@ def request(query, params):
|
|||
params['url'] = query_url
|
||||
|
||||
params['headers'].update(lang_info['headers'])
|
||||
params['headers']['Accept'] = (
|
||||
'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
|
||||
)
|
||||
params['headers']['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
|
||||
return params
|
||||
|
||||
|
||||
|
|
@ -128,8 +123,7 @@ def response(resp):
|
|||
# convert the text to dom
|
||||
dom = html.fromstring(resp.text)
|
||||
img_bas64_map = scrap_out_thumbs(dom)
|
||||
img_src_script = eval_xpath_getindex(
|
||||
dom, '//script[contains(., "AF_initDataCallback({key: ")]', 1).text
|
||||
img_src_script = eval_xpath_getindex(dom, '//script[contains(., "AF_initDataCallback({key: ")]', 1).text
|
||||
|
||||
# parse results
|
||||
#
|
||||
|
|
@ -189,7 +183,8 @@ def response(resp):
|
|||
if not src_url:
|
||||
src_url = thumbnail_src
|
||||
|
||||
results.append({
|
||||
results.append(
|
||||
{
|
||||
'url': url,
|
||||
'title': img_alt,
|
||||
'content': pub_descr,
|
||||
|
|
@ -197,7 +192,8 @@ def response(resp):
|
|||
'img_src': src_url,
|
||||
# 'img_format': img_format,
|
||||
'thumbnail_src': thumbnail_src,
|
||||
'template': 'images.html'
|
||||
})
|
||||
'template': 'images.html',
|
||||
}
|
||||
)
|
||||
|
||||
return results
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@ from searx.engines.google import (
|
|||
supported_languages_url,
|
||||
_fetch_supported_languages,
|
||||
)
|
||||
|
||||
# pylint: enable=unused-import
|
||||
|
||||
from searx.engines.google import (
|
||||
|
|
@ -71,14 +72,12 @@ time_range_support = True
|
|||
# safesearch : results are identitical for safesearch=0 and safesearch=2
|
||||
safesearch = False
|
||||
|
||||
|
||||
def request(query, params):
|
||||
"""Google-News search request"""
|
||||
|
||||
lang_info = get_lang_info(
|
||||
params, supported_languages, language_aliases, False
|
||||
)
|
||||
logger.debug(
|
||||
"HTTP header Accept-Language --> %s", lang_info['headers']['Accept-Language'])
|
||||
lang_info = get_lang_info(params, supported_languages, language_aliases, False)
|
||||
logger.debug("HTTP header Accept-Language --> %s", lang_info['headers']['Accept-Language'])
|
||||
|
||||
# google news has only one domain
|
||||
lang_info['subdomain'] = 'news.google.com'
|
||||
|
|
@ -94,19 +93,26 @@ def request(query, params):
|
|||
if params['time_range']:
|
||||
query += ' ' + time_range_dict[params['time_range']]
|
||||
|
||||
query_url = 'https://' + lang_info['subdomain'] + '/search' + "?" + urlencode({
|
||||
query_url = (
|
||||
'https://'
|
||||
+ lang_info['subdomain']
|
||||
+ '/search'
|
||||
+ "?"
|
||||
+ urlencode(
|
||||
{
|
||||
'q': query,
|
||||
**lang_info['params'],
|
||||
'ie': "utf8",
|
||||
'oe': "utf8",
|
||||
'gl': lang_info['country'],
|
||||
}) + ('&ceid=%s' % ceid) # ceid includes a ':' character which must not be urlencoded
|
||||
}
|
||||
)
|
||||
+ ('&ceid=%s' % ceid)
|
||||
) # ceid includes a ':' character which must not be urlencoded
|
||||
params['url'] = query_url
|
||||
|
||||
params['headers'].update(lang_info['headers'])
|
||||
params['headers']['Accept'] = (
|
||||
'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
|
||||
)
|
||||
params['headers']['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
|
||||
params['headers']['Cookie'] = "CONSENT=YES+cb.%s-14-p0.en+F+941;" % datetime.now().strftime("%Y%m%d")
|
||||
|
||||
return params
|
||||
|
|
@ -178,12 +184,14 @@ def response(resp):
|
|||
|
||||
img_src = extract_text(result.xpath('preceding-sibling::a/figure/img/@src'))
|
||||
|
||||
results.append({
|
||||
results.append(
|
||||
{
|
||||
'url': url,
|
||||
'title': title,
|
||||
'content': content,
|
||||
'img_src': img_src,
|
||||
})
|
||||
}
|
||||
)
|
||||
|
||||
# return results
|
||||
return results
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@ from searx.engines.google import (
|
|||
supported_languages_url,
|
||||
_fetch_supported_languages,
|
||||
)
|
||||
|
||||
# pylint: enable=unused-import
|
||||
|
||||
# about
|
||||
|
|
@ -52,6 +53,7 @@ use_locale_domain = True
|
|||
time_range_support = True
|
||||
safesearch = False
|
||||
|
||||
|
||||
def time_range_url(params):
|
||||
"""Returns a URL query component for a google-Scholar time range based on
|
||||
``params['time_range']``. Google-Scholar does only support ranges in years.
|
||||
|
|
@ -72,34 +74,38 @@ def request(query, params):
|
|||
"""Google-Scholar search request"""
|
||||
|
||||
offset = (params['pageno'] - 1) * 10
|
||||
lang_info = get_lang_info(
|
||||
params, supported_languages, language_aliases, False
|
||||
)
|
||||
logger.debug(
|
||||
"HTTP header Accept-Language --> %s", lang_info['headers']['Accept-Language'])
|
||||
lang_info = get_lang_info(params, supported_languages, language_aliases, False)
|
||||
logger.debug("HTTP header Accept-Language --> %s", lang_info['headers']['Accept-Language'])
|
||||
|
||||
# subdomain is: scholar.google.xy
|
||||
lang_info['subdomain'] = lang_info['subdomain'].replace("www.", "scholar.")
|
||||
|
||||
query_url = 'https://'+ lang_info['subdomain'] + '/scholar' + "?" + urlencode({
|
||||
query_url = (
|
||||
'https://'
|
||||
+ lang_info['subdomain']
|
||||
+ '/scholar'
|
||||
+ "?"
|
||||
+ urlencode(
|
||||
{
|
||||
'q': query,
|
||||
**lang_info['params'],
|
||||
'ie': "utf8",
|
||||
'oe': "utf8",
|
||||
'start': offset,
|
||||
})
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
query_url += time_range_url(params)
|
||||
params['url'] = query_url
|
||||
|
||||
params['headers'].update(lang_info['headers'])
|
||||
params['headers']['Accept'] = (
|
||||
'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
|
||||
)
|
||||
params['headers']['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
|
||||
|
||||
# params['google_subdomain'] = subdomain
|
||||
return params
|
||||
|
||||
|
||||
def response(resp):
|
||||
"""Get response from google's search request"""
|
||||
results = []
|
||||
|
|
@ -132,11 +138,13 @@ def response(resp):
|
|||
if pub_type:
|
||||
title = title + " " + pub_type
|
||||
|
||||
results.append({
|
||||
results.append(
|
||||
{
|
||||
'url': url,
|
||||
'title': title,
|
||||
'content': content,
|
||||
})
|
||||
}
|
||||
)
|
||||
|
||||
# parse suggestion
|
||||
for suggestion in eval_xpath(dom, '//div[contains(@class, "gs_qsuggest_wrap")]//li//a'):
|
||||
|
|
|
|||
|
|
@ -38,10 +38,8 @@ from searx.engines.google import (
|
|||
)
|
||||
|
||||
# pylint: disable=unused-import
|
||||
from searx.engines.google import (
|
||||
supported_languages_url
|
||||
, _fetch_supported_languages
|
||||
)
|
||||
from searx.engines.google import supported_languages_url, _fetch_supported_languages
|
||||
|
||||
# pylint: enable=unused-import
|
||||
|
||||
# about
|
||||
|
|
@ -65,6 +63,7 @@ safesearch = True
|
|||
|
||||
RE_CACHE = {}
|
||||
|
||||
|
||||
def _re(regexpr):
|
||||
"""returns compiled regular expression"""
|
||||
RE_CACHE[regexpr] = RE_CACHE.get(regexpr, re.compile(regexpr))
|
||||
|
|
@ -87,8 +86,7 @@ def scrap_out_thumbs_src(dom):
|
|||
|
||||
|
||||
def scrap_out_thumbs(dom):
|
||||
"""Scrap out thumbnail data from <script> tags.
|
||||
"""
|
||||
"""Scrap out thumbnail data from <script> tags."""
|
||||
ret_val = {}
|
||||
thumb_name = 'dimg_'
|
||||
|
||||
|
|
@ -112,19 +110,24 @@ def scrap_out_thumbs(dom):
|
|||
def request(query, params):
|
||||
"""Google-Video search request"""
|
||||
|
||||
lang_info = get_lang_info(
|
||||
params, supported_languages, language_aliases, False
|
||||
)
|
||||
logger.debug(
|
||||
"HTTP header Accept-Language --> %s", lang_info['headers']['Accept-Language'])
|
||||
lang_info = get_lang_info(params, supported_languages, language_aliases, False)
|
||||
logger.debug("HTTP header Accept-Language --> %s", lang_info['headers']['Accept-Language'])
|
||||
|
||||
query_url = 'https://' + lang_info['subdomain'] + '/search' + "?" + urlencode({
|
||||
query_url = (
|
||||
'https://'
|
||||
+ lang_info['subdomain']
|
||||
+ '/search'
|
||||
+ "?"
|
||||
+ urlencode(
|
||||
{
|
||||
'q': query,
|
||||
'tbm': "vid",
|
||||
**lang_info['params'],
|
||||
'ie': "utf8",
|
||||
'oe': "utf8",
|
||||
})
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
if params['time_range'] in time_range_dict:
|
||||
query_url += '&' + urlencode({'tbs': 'qdr:' + time_range_dict[params['time_range']]})
|
||||
|
|
@ -133,9 +136,7 @@ def request(query, params):
|
|||
params['url'] = query_url
|
||||
|
||||
params['headers'].update(lang_info['headers'])
|
||||
params['headers']['Accept'] = (
|
||||
'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
|
||||
)
|
||||
params['headers']['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
|
||||
return params
|
||||
|
||||
|
||||
|
|
@ -171,13 +172,13 @@ def response(resp):
|
|||
|
||||
title = extract_text(eval_xpath_getindex(result, title_xpath, 0))
|
||||
url = eval_xpath_getindex(result, './/div[@class="dXiKIc"]//a/@href', 0)
|
||||
length = extract_text(eval_xpath(
|
||||
result, './/div[contains(@class, "P7xzyf")]/span/span'))
|
||||
length = extract_text(eval_xpath(result, './/div[contains(@class, "P7xzyf")]/span/span'))
|
||||
c_node = eval_xpath_getindex(result, './/div[@class="Uroaid"]', 0)
|
||||
content = extract_text(c_node)
|
||||
pub_info = extract_text(eval_xpath(result, './/div[@class="Zg1NU"]'))
|
||||
|
||||
results.append({
|
||||
results.append(
|
||||
{
|
||||
'url': url,
|
||||
'title': title,
|
||||
'content': content,
|
||||
|
|
@ -185,7 +186,8 @@ def response(resp):
|
|||
'author': pub_info,
|
||||
'thumbnail': img_src,
|
||||
'template': 'videos.html',
|
||||
})
|
||||
}
|
||||
)
|
||||
|
||||
# parse suggestion
|
||||
for suggestion in eval_xpath_list(dom, suggestion_xpath):
|
||||
|
|
|
|||
|
|
@ -27,7 +27,9 @@ about = {
|
|||
"results": 'HTML',
|
||||
}
|
||||
|
||||
categories = ['general', ]
|
||||
categories = [
|
||||
'general',
|
||||
]
|
||||
paging = False
|
||||
|
||||
# suggestion_url = "https://sg.media-imdb.com/suggestion/{letter}/{query}.json"
|
||||
|
|
@ -35,13 +37,7 @@ suggestion_url = "https://v2.sg.media-imdb.com/suggestion/{letter}/{query}.json"
|
|||
|
||||
href_base = 'https://imdb.com/{category}/{entry_id}'
|
||||
|
||||
search_categories = {
|
||||
"nm": "name",
|
||||
"tt": "title",
|
||||
"kw": "keyword",
|
||||
"co": "company",
|
||||
"ep": "episode"
|
||||
}
|
||||
search_categories = {"nm": "name", "tt": "title", "kw": "keyword", "co": "company", "ep": "episode"}
|
||||
|
||||
|
||||
def request(query, params):
|
||||
|
|
@ -63,9 +59,7 @@ def response(resp):
|
|||
entry_id = entry['id']
|
||||
categ = search_categories.get(entry_id[:2])
|
||||
if categ is None:
|
||||
logger.error(
|
||||
'skip unknown category tag %s in %s', entry_id[:2], entry_id
|
||||
)
|
||||
logger.error('skip unknown category tag %s in %s', entry_id[:2], entry_id)
|
||||
continue
|
||||
|
||||
title = entry['l']
|
||||
|
|
@ -95,11 +89,13 @@ def response(resp):
|
|||
if not image_url_name.endswith('_V1_'):
|
||||
magic = '_V1_' + magic
|
||||
image_url = image_url_name + magic + '.' + image_url_prefix
|
||||
results.append({
|
||||
results.append(
|
||||
{
|
||||
"title": title,
|
||||
"url": href_base.format(category=categ, entry_id=entry_id),
|
||||
"content": content,
|
||||
"img_src": image_url,
|
||||
})
|
||||
}
|
||||
)
|
||||
|
||||
return results
|
||||
|
|
|
|||
|
|
@ -41,9 +41,7 @@ content_xpath = './/p[@class="media-body__summary"]'
|
|||
|
||||
# do search-request
|
||||
def request(query, params):
|
||||
params['url'] = search_url.format(ps=page_size,
|
||||
start=params['pageno'] * page_size,
|
||||
query=urlencode({'q': query}))
|
||||
params['url'] = search_url.format(ps=page_size, start=params['pageno'] * page_size, query=urlencode({'q': query}))
|
||||
|
||||
return params
|
||||
|
||||
|
|
@ -75,12 +73,16 @@ def response(resp):
|
|||
content = extract_text(result.xpath(content_xpath))
|
||||
|
||||
# append result
|
||||
results.append({'url': url,
|
||||
results.append(
|
||||
{
|
||||
'url': url,
|
||||
'title': title,
|
||||
'content': content,
|
||||
'template': 'videos.html',
|
||||
'publishedDate': publishedDate,
|
||||
'thumbnail': thumbnail})
|
||||
'thumbnail': thumbnail,
|
||||
}
|
||||
)
|
||||
|
||||
# return results
|
||||
return results
|
||||
|
|
|
|||
|
|
@ -46,14 +46,10 @@ def request(query, params):
|
|||
base_url_rand = base_url
|
||||
|
||||
search_url = base_url_rand + "api/v1/search?q={query}"
|
||||
params["url"] = search_url.format(
|
||||
query=quote_plus(query)
|
||||
) + "&page={pageno}".format(pageno=params["pageno"])
|
||||
params["url"] = search_url.format(query=quote_plus(query)) + "&page={pageno}".format(pageno=params["pageno"])
|
||||
|
||||
if params["time_range"] in time_range_dict:
|
||||
params["url"] += "&date={timerange}".format(
|
||||
timerange=time_range_dict[params["time_range"]]
|
||||
)
|
||||
params["url"] += "&date={timerange}".format(timerange=time_range_dict[params["time_range"]])
|
||||
|
||||
if params["language"] != "all":
|
||||
lang = params["language"].split("-")
|
||||
|
|
@ -88,17 +84,13 @@ def response(resp):
|
|||
url = base_invidious_url + videoid
|
||||
embedded = embedded_url.format(videoid=videoid)
|
||||
thumbs = result.get("videoThumbnails", [])
|
||||
thumb = next(
|
||||
(th for th in thumbs if th["quality"] == "sddefault"), None
|
||||
)
|
||||
thumb = next((th for th in thumbs if th["quality"] == "sddefault"), None)
|
||||
if thumb:
|
||||
thumbnail = thumb.get("url", "")
|
||||
else:
|
||||
thumbnail = ""
|
||||
|
||||
publishedDate = parser.parse(
|
||||
time.ctime(result.get("published", 0))
|
||||
)
|
||||
publishedDate = parser.parse(time.ctime(result.get("published", 0)))
|
||||
length = time.gmtime(result.get("lengthSeconds"))
|
||||
if length.tm_hour:
|
||||
length = time.strftime("%H:%M:%S", length)
|
||||
|
|
|
|||
|
|
@ -119,22 +119,22 @@ def response(resp):
|
|||
content = query(result, content_query)[0]
|
||||
except:
|
||||
content = ""
|
||||
results.append({
|
||||
results.append(
|
||||
{
|
||||
'url': to_string(url),
|
||||
'title': title_filter(to_string(title)),
|
||||
'content': content_filter(to_string(content)),
|
||||
})
|
||||
}
|
||||
)
|
||||
else:
|
||||
for url, title, content in zip(
|
||||
query(json, url_query),
|
||||
query(json, title_query),
|
||||
query(json, content_query)
|
||||
):
|
||||
results.append({
|
||||
for url, title, content in zip(query(json, url_query), query(json, title_query), query(json, content_query)):
|
||||
results.append(
|
||||
{
|
||||
'url': to_string(url),
|
||||
'title': title_filter(to_string(title)),
|
||||
'content': content_filter(to_string(content)),
|
||||
})
|
||||
}
|
||||
)
|
||||
|
||||
if not suggestion_query:
|
||||
return results
|
||||
|
|
|
|||
|
|
@ -34,8 +34,7 @@ content_xpath = './/span[@class="font11px lightgrey block"]'
|
|||
|
||||
# do search-request
|
||||
def request(query, params):
|
||||
params['url'] = search_url.format(search_term=quote(query),
|
||||
pageno=params['pageno'])
|
||||
params['url'] = search_url.format(search_term=quote(query), pageno=params['pageno'])
|
||||
|
||||
return params
|
||||
|
||||
|
|
@ -79,7 +78,9 @@ def response(resp):
|
|||
torrentfileurl = quote(torrentfile, safe="%/:=&?~#+!$,;'@()*")
|
||||
|
||||
# append result
|
||||
results.append({'url': href,
|
||||
results.append(
|
||||
{
|
||||
'url': href,
|
||||
'title': title,
|
||||
'content': content,
|
||||
'seed': seed,
|
||||
|
|
@ -88,7 +89,9 @@ def response(resp):
|
|||
'files': files,
|
||||
'magnetlink': magnetlink,
|
||||
'torrentfile': torrentfileurl,
|
||||
'template': 'torrent.html'})
|
||||
'template': 'torrent.html',
|
||||
}
|
||||
)
|
||||
|
||||
# return results sorted by seeder
|
||||
return sorted(results, key=itemgetter('seed'), reverse=True)
|
||||
|
|
|
|||
|
|
@ -34,9 +34,7 @@ IMG_SRC_FIXES = {
|
|||
|
||||
def request(query, params):
|
||||
|
||||
search_path = search_string.format(
|
||||
query=urlencode({'q': query}),
|
||||
page=params['pageno'])
|
||||
search_path = search_string.format(query=urlencode({'q': query}), page=params['pageno'])
|
||||
|
||||
params['url'] = base_url + search_path
|
||||
|
||||
|
|
@ -56,13 +54,15 @@ def response(resp):
|
|||
break
|
||||
else:
|
||||
img_src = result['image']['thumb']
|
||||
results.append({
|
||||
results.append(
|
||||
{
|
||||
'url': result['links']['item'],
|
||||
'title': result['title'],
|
||||
'img_src': img_src,
|
||||
'thumbnail_src': result['image']['thumb'],
|
||||
'author': result['creator'],
|
||||
'template': 'images.html'
|
||||
})
|
||||
'template': 'images.html',
|
||||
}
|
||||
)
|
||||
|
||||
return results
|
||||
|
|
|
|||
|
|
@ -22,29 +22,33 @@ paging = True
|
|||
time_range_support = False
|
||||
safesearch = False
|
||||
|
||||
|
||||
def request(query, params):
|
||||
|
||||
params['url'] = 'https://mediathekviewweb.de/api/query'
|
||||
params['method'] = 'POST'
|
||||
params['headers']['Content-type'] = 'text/plain'
|
||||
params['data'] = dumps({
|
||||
params['data'] = dumps(
|
||||
{
|
||||
'queries': [
|
||||
{
|
||||
'fields': [
|
||||
'title',
|
||||
'topic',
|
||||
],
|
||||
'query' : query
|
||||
'query': query,
|
||||
},
|
||||
],
|
||||
'sortBy': 'timestamp',
|
||||
'sortOrder': 'desc',
|
||||
'future': True,
|
||||
'offset': (params['pageno'] - 1) * 10,
|
||||
'size' : 10
|
||||
})
|
||||
'size': 10,
|
||||
}
|
||||
)
|
||||
return params
|
||||
|
||||
|
||||
def response(resp):
|
||||
|
||||
resp = loads(resp.text)
|
||||
|
|
@ -58,11 +62,13 @@ def response(resp):
|
|||
|
||||
item['hms'] = str(datetime.timedelta(seconds=item['duration']))
|
||||
|
||||
results.append({
|
||||
results.append(
|
||||
{
|
||||
'url': item['url_video_hd'],
|
||||
'title': "%(channel)s: %(title)s (%(hms)s)" % item,
|
||||
'length': item['hms'],
|
||||
'content': "%(description)s" % item,
|
||||
})
|
||||
}
|
||||
)
|
||||
|
||||
return results
|
||||
|
|
|
|||
|
|
@ -25,23 +25,24 @@ search_type = 'nearmatch' # possible values: title, text, nearmatch
|
|||
|
||||
# search-url
|
||||
base_url = 'https://{language}.wikipedia.org/'
|
||||
search_postfix = 'w/api.php?action=query'\
|
||||
'&list=search'\
|
||||
'&{query}'\
|
||||
'&format=json'\
|
||||
'&sroffset={offset}'\
|
||||
'&srlimit={limit}'\
|
||||
search_postfix = (
|
||||
'w/api.php?action=query'
|
||||
'&list=search'
|
||||
'&{query}'
|
||||
'&format=json'
|
||||
'&sroffset={offset}'
|
||||
'&srlimit={limit}'
|
||||
'&srwhat={searchtype}'
|
||||
)
|
||||
|
||||
|
||||
# do search-request
|
||||
def request(query, params):
|
||||
offset = (params['pageno'] - 1) * number_of_results
|
||||
|
||||
string_args = dict(query=urlencode({'srsearch': query}),
|
||||
offset=offset,
|
||||
limit=number_of_results,
|
||||
searchtype=search_type)
|
||||
string_args = dict(
|
||||
query=urlencode({'srsearch': query}), offset=offset, limit=number_of_results, searchtype=search_type
|
||||
)
|
||||
|
||||
format_strings = list(Formatter().parse(base_url))
|
||||
|
||||
|
|
@ -78,13 +79,14 @@ def response(resp):
|
|||
for result in search_results['query']['search']:
|
||||
if result.get('snippet', '').startswith('#REDIRECT'):
|
||||
continue
|
||||
url = base_url.format(language=resp.search_params['language']) +\
|
||||
'wiki/' + quote(result['title'].replace(' ', '_').encode())
|
||||
url = (
|
||||
base_url.format(language=resp.search_params['language'])
|
||||
+ 'wiki/'
|
||||
+ quote(result['title'].replace(' ', '_').encode())
|
||||
)
|
||||
|
||||
# append result
|
||||
results.append({'url': url,
|
||||
'title': result['title'],
|
||||
'content': ''})
|
||||
results.append({'url': url, 'title': result['title'], 'content': ''})
|
||||
|
||||
# return results
|
||||
return results
|
||||
|
|
|
|||
|
|
@ -26,7 +26,8 @@ def request(query, params):
|
|||
params['url'] = search_url
|
||||
params['method'] = 'POST'
|
||||
params['headers']['content-type'] = 'application/json; charset=utf-8'
|
||||
params['data'] = dumps({
|
||||
params['data'] = dumps(
|
||||
{
|
||||
'query': query,
|
||||
'queryExpression': '',
|
||||
'filters': [],
|
||||
|
|
@ -36,7 +37,8 @@ def request(query, params):
|
|||
'take': 10,
|
||||
'includeCitationContexts': False,
|
||||
'profileId': '',
|
||||
})
|
||||
}
|
||||
)
|
||||
|
||||
return params
|
||||
|
||||
|
|
@ -54,11 +56,13 @@ def response(resp):
|
|||
title = result['paper']['dn']
|
||||
content = _get_content(result['paper'])
|
||||
url = _paper_url.format(id=result['paper']['id'])
|
||||
results.append({
|
||||
results.append(
|
||||
{
|
||||
'url': url,
|
||||
'title': html_to_text(title),
|
||||
'content': html_to_text(content),
|
||||
})
|
||||
}
|
||||
)
|
||||
|
||||
return results
|
||||
|
||||
|
|
|
|||
|
|
@ -25,16 +25,17 @@ paging = True
|
|||
url = 'https://api.mixcloud.com/'
|
||||
search_url = url + 'search/?{query}&type=cloudcast&limit=10&offset={offset}'
|
||||
|
||||
embedded_url = '<iframe scrolling="no" frameborder="0" allowTransparency="true" ' +\
|
||||
'data-src="https://www.mixcloud.com/widget/iframe/?feed={url}" width="300" height="300"></iframe>'
|
||||
embedded_url = (
|
||||
'<iframe scrolling="no" frameborder="0" allowTransparency="true" '
|
||||
+ 'data-src="https://www.mixcloud.com/widget/iframe/?feed={url}" width="300" height="300"></iframe>'
|
||||
)
|
||||
|
||||
|
||||
# do search-request
|
||||
def request(query, params):
|
||||
offset = (params['pageno'] - 1) * 10
|
||||
|
||||
params['url'] = search_url.format(query=urlencode({'q': query}),
|
||||
offset=offset)
|
||||
params['url'] = search_url.format(query=urlencode({'q': query}), offset=offset)
|
||||
|
||||
return params
|
||||
|
||||
|
|
@ -54,11 +55,9 @@ def response(resp):
|
|||
publishedDate = parser.parse(result['created_time'])
|
||||
|
||||
# append result
|
||||
results.append({'url': url,
|
||||
'title': title,
|
||||
'embedded': embedded,
|
||||
'publishedDate': publishedDate,
|
||||
'content': content})
|
||||
results.append(
|
||||
{'url': url, 'title': title, 'embedded': embedded, 'publishedDate': publishedDate, 'content': content}
|
||||
)
|
||||
|
||||
# return results
|
||||
return results
|
||||
|
|
|
|||
|
|
@ -26,9 +26,11 @@ result_template = 'key-value.html'
|
|||
|
||||
_client = None
|
||||
|
||||
|
||||
def init(_):
|
||||
connect()
|
||||
|
||||
|
||||
def connect():
|
||||
global _client # pylint: disable=global-statement
|
||||
kwargs = {'port': port}
|
||||
|
|
@ -38,6 +40,7 @@ def connect():
|
|||
kwargs['password'] = password
|
||||
_client = MongoClient(host, **kwargs)[database][collection]
|
||||
|
||||
|
||||
def search(query, params):
|
||||
results = []
|
||||
if exact_match_only:
|
||||
|
|
@ -46,13 +49,7 @@ def search(query, params):
|
|||
_re = re.compile('.*{0}.*'.format(re.escape(query)), re.I | re.M)
|
||||
q = {'$regex': _re}
|
||||
|
||||
query = _client.find(
|
||||
{key: q}
|
||||
).skip(
|
||||
( params['pageno'] -1 ) * results_per_page
|
||||
).limit(
|
||||
results_per_page
|
||||
)
|
||||
query = _client.find({key: q}).skip((params['pageno'] - 1) * results_per_page).limit(results_per_page)
|
||||
|
||||
results.append({'number_of_results': query.count()})
|
||||
for r in query:
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ paging = True
|
|||
result_template = 'key-value.html'
|
||||
_connection = None
|
||||
|
||||
|
||||
def init(engine_settings):
|
||||
global _connection # pylint: disable=global-statement
|
||||
|
||||
|
|
@ -37,6 +38,7 @@ def init(engine_settings):
|
|||
auth_plugin=auth_plugin,
|
||||
)
|
||||
|
||||
|
||||
def search(query, params):
|
||||
query_params = {'query': query}
|
||||
query_to_run = query_str + ' LIMIT {0} OFFSET {1}'.format(limit, (params['pageno'] - 1) * limit)
|
||||
|
|
@ -46,6 +48,7 @@ def search(query, params):
|
|||
|
||||
return _fetch_results(cur)
|
||||
|
||||
|
||||
def _fetch_results(cur):
|
||||
results = []
|
||||
for res in cur:
|
||||
|
|
|
|||
|
|
@ -98,7 +98,9 @@ def response(resp):
|
|||
content = 'Category: "{category}". Downloaded {downloads} times.'
|
||||
content = content.format(category=category, downloads=downloads)
|
||||
|
||||
results.append({'url': href,
|
||||
results.append(
|
||||
{
|
||||
'url': href,
|
||||
'title': title,
|
||||
'content': content,
|
||||
'seed': seed,
|
||||
|
|
@ -106,6 +108,8 @@ def response(resp):
|
|||
'filesize': filesize,
|
||||
'torrentfile': torrent_link,
|
||||
'magnetlink': magnet_link,
|
||||
'template': 'torrent.html'})
|
||||
'template': 'torrent.html',
|
||||
}
|
||||
)
|
||||
|
||||
return results
|
||||
|
|
|
|||
|
|
@ -151,10 +151,12 @@ def response(resp):
|
|||
user_language = resp.search_params['language']
|
||||
|
||||
if resp.search_params['route']:
|
||||
results.append({
|
||||
results.append(
|
||||
{
|
||||
'answer': gettext('Get directions'),
|
||||
'url': route_url.format(*resp.search_params['route'].groups()),
|
||||
})
|
||||
}
|
||||
)
|
||||
|
||||
fetch_wikidata(nominatim_json, user_language)
|
||||
|
||||
|
|
@ -170,7 +172,8 @@ def response(resp):
|
|||
links, link_keys = get_links(result, user_language)
|
||||
data = get_data(result, user_language, link_keys)
|
||||
|
||||
results.append({
|
||||
results.append(
|
||||
{
|
||||
'template': 'map.html',
|
||||
'title': title,
|
||||
'address': address,
|
||||
|
|
@ -181,15 +184,14 @@ def response(resp):
|
|||
'img_src': img_src,
|
||||
'links': links,
|
||||
'data': data,
|
||||
'type': get_tag_label(
|
||||
result.get('category'), result.get('type', ''), user_language
|
||||
),
|
||||
'type': get_tag_label(result.get('category'), result.get('type', ''), user_language),
|
||||
'type_icon': result.get('icon'),
|
||||
'content': '',
|
||||
'longitude': result['lon'],
|
||||
'latitude': result['lat'],
|
||||
'boundingbox': result['boundingbox'],
|
||||
})
|
||||
}
|
||||
)
|
||||
|
||||
return results
|
||||
|
||||
|
|
@ -297,8 +299,7 @@ def get_title_address(result):
|
|||
|
||||
|
||||
def get_url_osm_geojson(result):
|
||||
"""Get url, osm and geojson
|
||||
"""
|
||||
"""Get url, osm and geojson"""
|
||||
osm_type = result.get('osm_type', result.get('type'))
|
||||
if 'osm_id' not in result:
|
||||
# see https://github.com/osm-search/Nominatim/issues/1521
|
||||
|
|
@ -349,11 +350,13 @@ def get_links(result, user_language):
|
|||
url, url_label = mapping_function(raw_value)
|
||||
if url.startswith('https://wikidata.org'):
|
||||
url_label = result.get('wikidata', {}).get('itemLabel') or url_label
|
||||
links.append({
|
||||
links.append(
|
||||
{
|
||||
'label': get_key_label(k, user_language),
|
||||
'url': url,
|
||||
'url_label': url_label,
|
||||
})
|
||||
}
|
||||
)
|
||||
link_keys.add(k)
|
||||
return links, link_keys
|
||||
|
||||
|
|
@ -373,11 +376,13 @@ def get_data(result, user_language, ignore_keys):
|
|||
continue
|
||||
k_label = get_key_label(k, user_language)
|
||||
if k_label:
|
||||
data.append({
|
||||
data.append(
|
||||
{
|
||||
'label': k_label,
|
||||
'key': k,
|
||||
'value': v,
|
||||
})
|
||||
}
|
||||
)
|
||||
data.sort(key=lambda entry: (get_key_rank(entry['key']), entry['label']))
|
||||
return data
|
||||
|
||||
|
|
|
|||
|
|
@ -34,10 +34,7 @@ def request(query, params):
|
|||
|
||||
params['url'] = pdbe_solr_url
|
||||
params['method'] = 'POST'
|
||||
params['data'] = {
|
||||
'q': query,
|
||||
'wt': "json" # request response in parsable format
|
||||
}
|
||||
params['data'] = {'q': query, 'wt': "json"} # request response in parsable format
|
||||
return params
|
||||
|
||||
|
||||
|
|
@ -53,12 +50,21 @@ def construct_body(result):
|
|||
if result['journal']:
|
||||
content = content.format(
|
||||
title=result['citation_title'],
|
||||
authors=result['entry_author_list'][0], journal=result['journal'], volume=result['journal_volume'],
|
||||
page=result['journal_page'], year=result['citation_year'])
|
||||
authors=result['entry_author_list'][0],
|
||||
journal=result['journal'],
|
||||
volume=result['journal_volume'],
|
||||
page=result['journal_page'],
|
||||
year=result['citation_year'],
|
||||
)
|
||||
else:
|
||||
content = content.format(
|
||||
title=result['citation_title'],
|
||||
authors=result['entry_author_list'][0], journal='', volume='', page='', year=result['release_year'])
|
||||
authors=result['entry_author_list'][0],
|
||||
journal='',
|
||||
volume='',
|
||||
page='',
|
||||
year=result['release_year'],
|
||||
)
|
||||
img_src = pdbe_preview_url.format(pdb_id=result['pdb_id'])
|
||||
except (KeyError):
|
||||
content = None
|
||||
|
|
@ -96,20 +102,21 @@ def response(resp):
|
|||
# since we can't construct a proper body from the response, we'll make up our own
|
||||
msg_superseded = gettext("This entry has been superseded by")
|
||||
content = '{msg_superseded}: {url} ({pdb_id})'.format(
|
||||
msg_superseded=msg_superseded,
|
||||
url=superseded_url,
|
||||
pdb_id=result['superseded_by'])
|
||||
msg_superseded=msg_superseded, url=superseded_url, pdb_id=result['superseded_by']
|
||||
)
|
||||
|
||||
# obsoleted entries don't have preview images
|
||||
img_src = None
|
||||
else:
|
||||
title, content, img_src = construct_body(result)
|
||||
|
||||
results.append({
|
||||
results.append(
|
||||
{
|
||||
'url': pdbe_entry_url.format(pdb_id=result['pdb_id']),
|
||||
'title': title,
|
||||
'content': content,
|
||||
'img_src': img_src
|
||||
})
|
||||
'img_src': img_src,
|
||||
}
|
||||
)
|
||||
|
||||
return results
|
||||
|
|
|
|||
|
|
@ -36,9 +36,7 @@ def request(query, params):
|
|||
language = params["language"].split("-")[0]
|
||||
if "all" != language and language in supported_languages:
|
||||
query_dict["languageOneOf"] = language
|
||||
params["url"] = search_url.format(
|
||||
query=urlencode(query_dict), pageno=pageno
|
||||
)
|
||||
params["url"] = search_url.format(query=urlencode(query_dict), pageno=pageno)
|
||||
return params
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -33,9 +33,7 @@ supported_languages = ['de', 'en', 'fr', 'it']
|
|||
|
||||
# do search-request
|
||||
def request(query, params):
|
||||
params['url'] = base_url +\
|
||||
search_string.format(query=urlencode({'q': query}),
|
||||
limit=number_of_results)
|
||||
params['url'] = base_url + search_string.format(query=urlencode({'q': query}), limit=number_of_results)
|
||||
|
||||
if params['language'] != 'all':
|
||||
language = params['language'].split('_')[0]
|
||||
|
|
@ -75,50 +73,60 @@ def response(resp):
|
|||
# continue if invalide osm-type
|
||||
continue
|
||||
|
||||
url = result_base_url.format(osm_type=osm_type,
|
||||
osm_id=properties.get('osm_id'))
|
||||
url = result_base_url.format(osm_type=osm_type, osm_id=properties.get('osm_id'))
|
||||
|
||||
osm = {'type': osm_type,
|
||||
'id': properties.get('osm_id')}
|
||||
osm = {'type': osm_type, 'id': properties.get('osm_id')}
|
||||
|
||||
geojson = r.get('geometry')
|
||||
|
||||
if properties.get('extent'):
|
||||
boundingbox = [properties.get('extent')[3],
|
||||
boundingbox = [
|
||||
properties.get('extent')[3],
|
||||
properties.get('extent')[1],
|
||||
properties.get('extent')[0],
|
||||
properties.get('extent')[2]]
|
||||
properties.get('extent')[2],
|
||||
]
|
||||
else:
|
||||
# TODO: better boundingbox calculation
|
||||
boundingbox = [geojson['coordinates'][1],
|
||||
boundingbox = [
|
||||
geojson['coordinates'][1],
|
||||
geojson['coordinates'][1],
|
||||
geojson['coordinates'][0],
|
||||
geojson['coordinates'][0]]
|
||||
geojson['coordinates'][0],
|
||||
]
|
||||
|
||||
# address calculation
|
||||
address = {}
|
||||
|
||||
# get name
|
||||
if properties.get('osm_key') == 'amenity' or\
|
||||
properties.get('osm_key') == 'shop' or\
|
||||
properties.get('osm_key') == 'tourism' or\
|
||||
properties.get('osm_key') == 'leisure':
|
||||
if (
|
||||
properties.get('osm_key') == 'amenity'
|
||||
or properties.get('osm_key') == 'shop'
|
||||
or properties.get('osm_key') == 'tourism'
|
||||
or properties.get('osm_key') == 'leisure'
|
||||
):
|
||||
address = {'name': properties.get('name')}
|
||||
|
||||
# add rest of adressdata, if something is already found
|
||||
if address.get('name'):
|
||||
address.update({'house_number': properties.get('housenumber'),
|
||||
address.update(
|
||||
{
|
||||
'house_number': properties.get('housenumber'),
|
||||
'road': properties.get('street'),
|
||||
'locality': properties.get('city',
|
||||
properties.get('town', # noqa
|
||||
properties.get('village'))), # noqa
|
||||
'locality': properties.get(
|
||||
'city', properties.get('town', properties.get('village')) # noqa
|
||||
), # noqa
|
||||
'postcode': properties.get('postcode'),
|
||||
'country': properties.get('country')})
|
||||
'country': properties.get('country'),
|
||||
}
|
||||
)
|
||||
else:
|
||||
address = None
|
||||
|
||||
# append result
|
||||
results.append({'template': 'map.html',
|
||||
results.append(
|
||||
{
|
||||
'template': 'map.html',
|
||||
'title': title,
|
||||
'content': '',
|
||||
'longitude': geojson['coordinates'][0],
|
||||
|
|
@ -127,7 +135,9 @@ def response(resp):
|
|||
'geojson': geojson,
|
||||
'address': address,
|
||||
'osm': osm,
|
||||
'url': url})
|
||||
'url': url,
|
||||
}
|
||||
)
|
||||
|
||||
# return results
|
||||
return results
|
||||
|
|
|
|||
|
|
@ -40,17 +40,14 @@ trackers = [
|
|||
]
|
||||
|
||||
# piratebay specific type-definitions
|
||||
search_types = {"files": "0",
|
||||
"music": "100",
|
||||
"videos": "200"}
|
||||
search_types = {"files": "0", "music": "100", "videos": "200"}
|
||||
|
||||
|
||||
# do search-request
|
||||
def request(query, params):
|
||||
search_type = search_types.get(params["category"], "0")
|
||||
|
||||
params["url"] = search_url.format(search_term=quote(query),
|
||||
search_type=search_type)
|
||||
params["url"] = search_url.format(search_term=quote(query), search_type=search_type)
|
||||
|
||||
return params
|
||||
|
||||
|
|
@ -68,8 +65,9 @@ def response(resp):
|
|||
# parse results
|
||||
for result in search_res:
|
||||
link = url + "description.php?id=" + result["id"]
|
||||
magnetlink = "magnet:?xt=urn:btih:" + result["info_hash"] + "&dn=" + result["name"]\
|
||||
+ "&tr=" + "&tr=".join(trackers)
|
||||
magnetlink = (
|
||||
"magnet:?xt=urn:btih:" + result["info_hash"] + "&dn=" + result["name"] + "&tr=" + "&tr=".join(trackers)
|
||||
)
|
||||
|
||||
params = {
|
||||
"url": link,
|
||||
|
|
@ -77,7 +75,7 @@ def response(resp):
|
|||
"seed": result["seeders"],
|
||||
"leech": result["leechers"],
|
||||
"magnetlink": magnetlink,
|
||||
"template": "torrent.html"
|
||||
"template": "torrent.html",
|
||||
}
|
||||
|
||||
# extract and convert creation date
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ paging = True
|
|||
result_template = 'key-value.html'
|
||||
_connection = None
|
||||
|
||||
|
||||
def init(engine_settings):
|
||||
global _connection # pylint: disable=global-statement
|
||||
|
||||
|
|
@ -37,18 +38,17 @@ def init(engine_settings):
|
|||
port=port,
|
||||
)
|
||||
|
||||
|
||||
def search(query, params):
|
||||
query_params = {'query': query}
|
||||
query_to_run = (
|
||||
query_str
|
||||
+ ' LIMIT {0} OFFSET {1}'.format(limit, (params['pageno'] - 1) * limit)
|
||||
)
|
||||
query_to_run = query_str + ' LIMIT {0} OFFSET {1}'.format(limit, (params['pageno'] - 1) * limit)
|
||||
|
||||
with _connection:
|
||||
with _connection.cursor() as cur:
|
||||
cur.execute(query_to_run, query_params)
|
||||
return _fetch_results(cur)
|
||||
|
||||
|
||||
def _fetch_results(cur):
|
||||
results = []
|
||||
titles = []
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ about = {
|
|||
"wikidata_id": 'Q1540899',
|
||||
"official_api_documentation": {
|
||||
'url': 'https://www.ncbi.nlm.nih.gov/home/develop/api/',
|
||||
'comment': 'More info on api: https://www.ncbi.nlm.nih.gov/books/NBK25501/'
|
||||
'comment': 'More info on api: https://www.ncbi.nlm.nih.gov/books/NBK25501/',
|
||||
},
|
||||
"use_official_api": True,
|
||||
"require_api_key": False,
|
||||
|
|
@ -24,8 +24,9 @@ about = {
|
|||
|
||||
categories = ['science']
|
||||
|
||||
base_url = 'https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi'\
|
||||
+ '?db=pubmed&{query}&retstart={offset}&retmax={hits}'
|
||||
base_url = (
|
||||
'https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi' + '?db=pubmed&{query}&retstart={offset}&retmax={hits}'
|
||||
)
|
||||
|
||||
# engine dependent config
|
||||
number_of_results = 10
|
||||
|
|
@ -36,9 +37,7 @@ def request(query, params):
|
|||
# basic search
|
||||
offset = (params['pageno'] - 1) * number_of_results
|
||||
|
||||
string_args = dict(query=urlencode({'term': query}),
|
||||
offset=offset,
|
||||
hits=number_of_results)
|
||||
string_args = dict(query=urlencode({'term': query}), offset=offset, hits=number_of_results)
|
||||
|
||||
params['url'] = base_url.format(**string_args)
|
||||
|
||||
|
|
@ -49,8 +48,9 @@ def response(resp):
|
|||
results = []
|
||||
|
||||
# First retrieve notice of each result
|
||||
pubmed_retrieve_api_url = 'https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?'\
|
||||
+ 'db=pubmed&retmode=xml&id={pmids_string}'
|
||||
pubmed_retrieve_api_url = (
|
||||
'https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?' + 'db=pubmed&retmode=xml&id={pmids_string}'
|
||||
)
|
||||
|
||||
pmids_results = etree.XML(resp.content)
|
||||
pmids = pmids_results.xpath('//eSearchResult/IdList/Id')
|
||||
|
|
@ -88,14 +88,17 @@ def response(resp):
|
|||
content = content[0:300] + "..."
|
||||
# TODO: center snippet on query term
|
||||
|
||||
res_dict = {'url': url,
|
||||
'title': title,
|
||||
'content': content}
|
||||
res_dict = {'url': url, 'title': title, 'content': content}
|
||||
|
||||
try:
|
||||
publishedDate = datetime.strptime(entry.xpath('.//DateCreated/Year')[0].text
|
||||
+ '-' + entry.xpath('.//DateCreated/Month')[0].text
|
||||
+ '-' + entry.xpath('.//DateCreated/Day')[0].text, '%Y-%m-%d')
|
||||
publishedDate = datetime.strptime(
|
||||
entry.xpath('.//DateCreated/Year')[0].text
|
||||
+ '-'
|
||||
+ entry.xpath('.//DateCreated/Month')[0].text
|
||||
+ '-'
|
||||
+ entry.xpath('.//DateCreated/Day')[0].text,
|
||||
'%Y-%m-%d',
|
||||
)
|
||||
res_dict['publishedDate'] = publishedDate
|
||||
except:
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -61,6 +61,7 @@ category_to_keyword = {
|
|||
# search-url
|
||||
url = 'https://api.qwant.com/v3/search/{keyword}?{query}&count={count}&offset={offset}'
|
||||
|
||||
|
||||
def request(query, params):
|
||||
"""Qwant search request"""
|
||||
keyword = category_to_keyword[categories[0]]
|
||||
|
|
@ -111,7 +112,14 @@ def response(resp):
|
|||
|
||||
# check for an API error
|
||||
if search_results.get('status') != 'success':
|
||||
msg = ",".join(data.get('message', ['unknown', ]))
|
||||
msg = ",".join(
|
||||
data.get(
|
||||
'message',
|
||||
[
|
||||
'unknown',
|
||||
],
|
||||
)
|
||||
)
|
||||
raise SearxEngineAPIException('API error::' + msg)
|
||||
|
||||
# raise for other errors
|
||||
|
|
@ -153,11 +161,13 @@ def response(resp):
|
|||
|
||||
if mainline_type == 'web':
|
||||
content = item['desc']
|
||||
results.append({
|
||||
results.append(
|
||||
{
|
||||
'title': title,
|
||||
'url': res_url,
|
||||
'content': content,
|
||||
})
|
||||
}
|
||||
)
|
||||
|
||||
elif mainline_type == 'news':
|
||||
|
||||
|
|
@ -168,23 +178,27 @@ def response(resp):
|
|||
img_src = None
|
||||
if news_media:
|
||||
img_src = news_media[0].get('pict', {}).get('url', None)
|
||||
results.append({
|
||||
results.append(
|
||||
{
|
||||
'title': title,
|
||||
'url': res_url,
|
||||
'publishedDate': pub_date,
|
||||
'img_src': img_src,
|
||||
})
|
||||
}
|
||||
)
|
||||
|
||||
elif mainline_type == 'images':
|
||||
thumbnail = item['thumbnail']
|
||||
img_src = item['media']
|
||||
results.append({
|
||||
results.append(
|
||||
{
|
||||
'title': title,
|
||||
'url': res_url,
|
||||
'template': 'images.html',
|
||||
'thumbnail_src': thumbnail,
|
||||
'img_src': img_src,
|
||||
})
|
||||
}
|
||||
)
|
||||
|
||||
elif mainline_type == 'videos':
|
||||
# some videos do not have a description: while qwant-video
|
||||
|
|
@ -208,11 +222,9 @@ def response(resp):
|
|||
thumbnail = item['thumbnail']
|
||||
# from some locations (DE and others?) the s2 link do
|
||||
# response a 'Please wait ..' but does not deliver the thumbnail
|
||||
thumbnail = thumbnail.replace(
|
||||
'https://s2.qwant.com',
|
||||
'https://s1.qwant.com', 1
|
||||
)
|
||||
results.append({
|
||||
thumbnail = thumbnail.replace('https://s2.qwant.com', 'https://s1.qwant.com', 1)
|
||||
results.append(
|
||||
{
|
||||
'title': title,
|
||||
'url': res_url,
|
||||
'content': content,
|
||||
|
|
@ -220,7 +232,8 @@ def response(resp):
|
|||
'thumbnail': thumbnail,
|
||||
'template': 'videos.html',
|
||||
'length': length,
|
||||
})
|
||||
}
|
||||
)
|
||||
|
||||
return results
|
||||
|
||||
|
|
|
|||
|
|
@ -28,18 +28,12 @@ mount_prefix = None
|
|||
dl_prefix = None
|
||||
|
||||
# embedded
|
||||
embedded_url = '<{ttype} controls height="166px" ' +\
|
||||
'src="{url}" type="{mtype}"></{ttype}>'
|
||||
embedded_url = '<{ttype} controls height="166px" ' + 'src="{url}" type="{mtype}"></{ttype}>'
|
||||
|
||||
|
||||
# helper functions
|
||||
def get_time_range(time_range):
|
||||
sw = {
|
||||
'day': 1,
|
||||
'week': 7,
|
||||
'month': 30,
|
||||
'year': 365
|
||||
}
|
||||
sw = {'day': 1, 'week': 7, 'month': 30, 'year': 365}
|
||||
|
||||
offset = sw.get(time_range, 0)
|
||||
if not offset:
|
||||
|
|
@ -52,11 +46,9 @@ def get_time_range(time_range):
|
|||
def request(query, params):
|
||||
search_after = get_time_range(params['time_range'])
|
||||
search_url = base_url + 'json?{query}&highlight=0'
|
||||
params['url'] = search_url.format(query=urlencode({
|
||||
'query': query,
|
||||
'page': params['pageno'],
|
||||
'after': search_after,
|
||||
'dir': search_dir}))
|
||||
params['url'] = search_url.format(
|
||||
query=urlencode({'query': query, 'page': params['pageno'], 'after': search_after, 'dir': search_dir})
|
||||
)
|
||||
|
||||
return params
|
||||
|
||||
|
|
@ -76,10 +68,7 @@ def response(resp):
|
|||
content = '{}'.format(result['snippet'])
|
||||
|
||||
# append result
|
||||
item = {'url': url,
|
||||
'title': title,
|
||||
'content': content,
|
||||
'template': 'files.html'}
|
||||
item = {'url': url, 'title': title, 'content': content, 'template': 'files.html'}
|
||||
|
||||
if result['size']:
|
||||
item['size'] = int(result['size'])
|
||||
|
|
@ -96,9 +85,8 @@ def response(resp):
|
|||
|
||||
if mtype in ['audio', 'video']:
|
||||
item['embedded'] = embedded_url.format(
|
||||
ttype=mtype,
|
||||
url=quote(url.encode('utf8'), '/:'),
|
||||
mtype=result['mtype'])
|
||||
ttype=mtype, url=quote(url.encode('utf8'), '/:'), mtype=result['mtype']
|
||||
)
|
||||
|
||||
if mtype in ['image'] and subtype in ['bmp', 'gif', 'jpeg', 'png']:
|
||||
item['img_src'] = url
|
||||
|
|
|
|||
|
|
@ -52,10 +52,7 @@ def response(resp):
|
|||
data = post['data']
|
||||
|
||||
# extract post information
|
||||
params = {
|
||||
'url': urljoin(base_url, data['permalink']),
|
||||
'title': data['title']
|
||||
}
|
||||
params = {'url': urljoin(base_url, data['permalink']), 'title': data['title']}
|
||||
|
||||
# if thumbnail field contains a valid URL, we need to change template
|
||||
thumbnail = data['thumbnail']
|
||||
|
|
|
|||
|
|
@ -20,6 +20,8 @@ result_template = 'key-value.html'
|
|||
exact_match_only = True
|
||||
|
||||
_redis_client = None
|
||||
|
||||
|
||||
def init(_engine_settings):
|
||||
global _redis_client # pylint: disable=global-statement
|
||||
_redis_client = redis.StrictRedis(
|
||||
|
|
@ -30,6 +32,7 @@ def init(_engine_settings):
|
|||
decode_responses=True,
|
||||
)
|
||||
|
||||
|
||||
def search(query, _params):
|
||||
if not exact_match_only:
|
||||
return search_keys(query)
|
||||
|
|
@ -42,21 +45,20 @@ def search(query, _params):
|
|||
if ' ' in query:
|
||||
qset, rest = query.split(' ', 1)
|
||||
ret = []
|
||||
for res in _redis_client.hscan_iter(
|
||||
qset, match='*{}*'.format(rest)
|
||||
):
|
||||
ret.append({
|
||||
for res in _redis_client.hscan_iter(qset, match='*{}*'.format(rest)):
|
||||
ret.append(
|
||||
{
|
||||
res[0]: res[1],
|
||||
'template': result_template,
|
||||
})
|
||||
}
|
||||
)
|
||||
return ret
|
||||
return []
|
||||
|
||||
|
||||
def search_keys(query):
|
||||
ret = []
|
||||
for key in _redis_client.scan_iter(
|
||||
match='*{}*'.format(query)
|
||||
):
|
||||
for key in _redis_client.scan_iter(match='*{}*'.format(query)):
|
||||
key_type = _redis_client.type(key)
|
||||
res = None
|
||||
|
||||
|
|
|
|||
|
|
@ -68,7 +68,8 @@ def response(resp):
|
|||
else:
|
||||
content = f"{views} views - {rumbles} rumbles"
|
||||
|
||||
results.append({
|
||||
results.append(
|
||||
{
|
||||
'url': url,
|
||||
'title': title,
|
||||
'content': content,
|
||||
|
|
@ -77,5 +78,6 @@ def response(resp):
|
|||
'template': 'videos.html',
|
||||
'publishedDate': fixed_date,
|
||||
'thumbnail': thumbnail,
|
||||
})
|
||||
}
|
||||
)
|
||||
return results
|
||||
|
|
|
|||
|
|
@ -32,12 +32,16 @@ def request(query, params):
|
|||
params['url'] = search_url
|
||||
params['method'] = 'POST'
|
||||
params['headers']['Content-type'] = "application/json"
|
||||
params['data'] = dumps({"query": query,
|
||||
params['data'] = dumps(
|
||||
{
|
||||
"query": query,
|
||||
"searchField": "ALL",
|
||||
"sortDirection": "ASC",
|
||||
"sortOrder": "RELEVANCY",
|
||||
"page": params['pageno'],
|
||||
"pageSize": page_size})
|
||||
"pageSize": page_size,
|
||||
}
|
||||
)
|
||||
|
||||
return params
|
||||
|
||||
|
|
@ -69,11 +73,15 @@ def response(resp):
|
|||
content = result['highlights'][0]['value']
|
||||
|
||||
# append result
|
||||
results.append({'url': url + 'structure/' + result['id'],
|
||||
results.append(
|
||||
{
|
||||
'url': url + 'structure/' + result['id'],
|
||||
'title': result['label'],
|
||||
# 'thumbnail': thumbnail,
|
||||
'img_src': thumbnail,
|
||||
'content': html_to_text(content)})
|
||||
'content': html_to_text(content),
|
||||
}
|
||||
)
|
||||
|
||||
# return results
|
||||
return results
|
||||
|
|
|
|||
|
|
@ -25,10 +25,7 @@ url = 'https://searchcode.com/'
|
|||
search_url = url + 'api/codesearch_I/?{query}&p={pageno}'
|
||||
|
||||
# special code-endings which are not recognised by the file ending
|
||||
code_endings = {'cs': 'c#',
|
||||
'h': 'c',
|
||||
'hpp': 'cpp',
|
||||
'cxx': 'cpp'}
|
||||
code_endings = {'cs': 'c#', 'h': 'c', 'hpp': 'cpp', 'cxx': 'cpp'}
|
||||
|
||||
|
||||
# do search-request
|
||||
|
|
@ -55,17 +52,21 @@ def response(resp):
|
|||
lines[int(line)] = code
|
||||
|
||||
code_language = code_endings.get(
|
||||
result['filename'].split('.')[-1].lower(),
|
||||
result['filename'].split('.')[-1].lower())
|
||||
result['filename'].split('.')[-1].lower(), result['filename'].split('.')[-1].lower()
|
||||
)
|
||||
|
||||
# append result
|
||||
results.append({'url': href,
|
||||
results.append(
|
||||
{
|
||||
'url': href,
|
||||
'title': title,
|
||||
'content': '',
|
||||
'repository': repo,
|
||||
'codelines': sorted(lines.items()),
|
||||
'code_language': code_language,
|
||||
'template': 'code.html'})
|
||||
'template': 'code.html',
|
||||
}
|
||||
)
|
||||
|
||||
# return results
|
||||
return results
|
||||
|
|
|
|||
|
|
@ -37,7 +37,7 @@ def request(query, params):
|
|||
'language': params['language'],
|
||||
'time_range': params['time_range'],
|
||||
'category': params['category'],
|
||||
'format': 'json'
|
||||
'format': 'json',
|
||||
}
|
||||
|
||||
return params
|
||||
|
|
|
|||
|
|
@ -13,7 +13,8 @@ def request(query, params):
|
|||
params['url'] = search_url
|
||||
params['method'] = 'POST'
|
||||
params['headers']['content-type'] = 'application/json'
|
||||
params['data'] = dumps({
|
||||
params['data'] = dumps(
|
||||
{
|
||||
"queryString": query,
|
||||
"page": params['pageno'],
|
||||
"pageSize": 10,
|
||||
|
|
@ -25,7 +26,8 @@ def request(query, params):
|
|||
"coAuthors": [],
|
||||
"venues": [],
|
||||
"performTitleMatch": True,
|
||||
})
|
||||
}
|
||||
)
|
||||
return params
|
||||
|
||||
|
||||
|
|
@ -33,10 +35,12 @@ def response(resp):
|
|||
res = loads(resp.text)
|
||||
results = []
|
||||
for result in res['results']:
|
||||
results.append({
|
||||
results.append(
|
||||
{
|
||||
'url': result['primaryPaperLink']['url'],
|
||||
'title': result['title']['text'],
|
||||
'content': result['paperAbstractTruncated']
|
||||
})
|
||||
'content': result['paperAbstractTruncated'],
|
||||
}
|
||||
)
|
||||
|
||||
return results
|
||||
|
|
|
|||
|
|
@ -23,23 +23,21 @@ paging = True
|
|||
time_range_support = True
|
||||
safesearch = True
|
||||
supported_languages = [
|
||||
# fmt: off
|
||||
'en', 'fr', 'ja', 'eu', 'ca', 'cs', 'eo', 'el',
|
||||
'de', 'it', 'nl', 'es', 'oc', 'gd', 'zh', 'pt',
|
||||
'sv', 'pl', 'fi', 'ru'
|
||||
# fmt: on
|
||||
]
|
||||
base_url = 'https://sepiasearch.org/api/v1/search/videos'
|
||||
|
||||
safesearch_table = {
|
||||
0: 'both',
|
||||
1: 'false',
|
||||
2: 'false'
|
||||
}
|
||||
safesearch_table = {0: 'both', 1: 'false', 2: 'false'}
|
||||
|
||||
time_range_table = {
|
||||
'day': relativedelta.relativedelta(),
|
||||
'week': relativedelta.relativedelta(weeks=-1),
|
||||
'month': relativedelta.relativedelta(months=-1),
|
||||
'year': relativedelta.relativedelta(years=-1)
|
||||
'year': relativedelta.relativedelta(years=-1),
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -53,13 +51,19 @@ def minute_to_hm(minute):
|
|||
|
||||
|
||||
def request(query, params):
|
||||
params['url'] = base_url + '?' + urlencode({
|
||||
params['url'] = (
|
||||
base_url
|
||||
+ '?'
|
||||
+ urlencode(
|
||||
{
|
||||
'search': query,
|
||||
'start': (params['pageno'] - 1) * 10,
|
||||
'count': 10,
|
||||
'sort': '-match',
|
||||
'nsfw': safesearch_table[params['safesearch']]
|
||||
})
|
||||
'nsfw': safesearch_table[params['safesearch']],
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
language = params['language'].split('-')[0]
|
||||
if language in supported_languages:
|
||||
|
|
@ -89,7 +93,9 @@ def response(resp):
|
|||
length = minute_to_hm(result.get('duration'))
|
||||
url = result['url']
|
||||
|
||||
results.append({'url': url,
|
||||
results.append(
|
||||
{
|
||||
'url': url,
|
||||
'title': title,
|
||||
'content': content,
|
||||
'author': author,
|
||||
|
|
@ -97,6 +103,8 @@ def response(resp):
|
|||
'template': 'videos.html',
|
||||
'publishedDate': publishedDate,
|
||||
'embedded': embedded,
|
||||
'thumbnail': thumbnail})
|
||||
'thumbnail': thumbnail,
|
||||
}
|
||||
)
|
||||
|
||||
return results
|
||||
|
|
|
|||
|
|
@ -58,10 +58,12 @@ def response(resp):
|
|||
if result_data is None:
|
||||
continue
|
||||
title_element = eval_xpath_getindex(result_element, './/h3/a', 0)
|
||||
results.append({
|
||||
results.append(
|
||||
{
|
||||
'url': title_element.get('href'),
|
||||
'title': extract_text(title_element),
|
||||
'content': extract_text(eval_xpath(result_data, './/div[@class="_3eded7"]')),
|
||||
})
|
||||
}
|
||||
)
|
||||
|
||||
return results
|
||||
|
|
|
|||
|
|
@ -28,9 +28,11 @@ URL = 'https://sjp.pwn.pl'
|
|||
SEARCH_URL = URL + '/szukaj/{query}.html'
|
||||
|
||||
word_xpath = '//div[@class="query"]'
|
||||
dict_xpath = ['//div[@class="wyniki sjp-so-wyniki sjp-so-anchor"]',
|
||||
dict_xpath = [
|
||||
'//div[@class="wyniki sjp-so-wyniki sjp-so-anchor"]',
|
||||
'//div[@class="wyniki sjp-wyniki sjp-anchor"]',
|
||||
'//div[@class="wyniki sjp-doroszewski-wyniki sjp-doroszewski-anchor"]']
|
||||
'//div[@class="wyniki sjp-doroszewski-wyniki sjp-doroszewski-anchor"]',
|
||||
]
|
||||
|
||||
|
||||
def request(query, params):
|
||||
|
|
@ -85,9 +87,11 @@ def response(resp):
|
|||
infobox += "</ol>"
|
||||
infobox += "</ul></div>"
|
||||
|
||||
results.append({
|
||||
results.append(
|
||||
{
|
||||
'infobox': word,
|
||||
'content': infobox,
|
||||
})
|
||||
}
|
||||
)
|
||||
|
||||
return results
|
||||
|
|
|
|||
|
|
@ -36,7 +36,8 @@ def response(resp):
|
|||
search_results = loads(resp.text)
|
||||
|
||||
for result in search_results["results"]:
|
||||
results.append({
|
||||
results.append(
|
||||
{
|
||||
'infohash': result["infohash"],
|
||||
'seed': result["swarm"]["seeders"],
|
||||
'leech': result["swarm"]["leechers"],
|
||||
|
|
@ -45,5 +46,6 @@ def response(resp):
|
|||
'filesize': result["size"],
|
||||
'magnetlink': result["magnet"],
|
||||
'template': "torrent.html",
|
||||
})
|
||||
}
|
||||
)
|
||||
return results
|
||||
|
|
|
|||
|
|
@ -27,17 +27,21 @@ paging = True
|
|||
# search-url
|
||||
# missing attribute: user_id, app_version, app_locale
|
||||
url = 'https://api-v2.soundcloud.com/'
|
||||
search_url = url + 'search?{query}'\
|
||||
'&variant_ids='\
|
||||
'&facet=model'\
|
||||
'&limit=20'\
|
||||
'&offset={offset}'\
|
||||
'&linked_partitioning=1'\
|
||||
'&client_id={client_id}' # noqa
|
||||
search_url = (
|
||||
url + 'search?{query}'
|
||||
'&variant_ids='
|
||||
'&facet=model'
|
||||
'&limit=20'
|
||||
'&offset={offset}'
|
||||
'&linked_partitioning=1'
|
||||
'&client_id={client_id}'
|
||||
) # noqa
|
||||
|
||||
embedded_url = '<iframe width="100%" height="166" ' +\
|
||||
'scrolling="no" frameborder="no" ' +\
|
||||
'data-src="https://w.soundcloud.com/player/?url={uri}"></iframe>'
|
||||
embedded_url = (
|
||||
'<iframe width="100%" height="166" '
|
||||
+ 'scrolling="no" frameborder="no" '
|
||||
+ 'data-src="https://w.soundcloud.com/player/?url={uri}"></iframe>'
|
||||
)
|
||||
|
||||
cid_re = re.compile(r'client_id:"([^"]*)"', re.I | re.U)
|
||||
guest_client_id = ''
|
||||
|
|
@ -75,9 +79,7 @@ def init(engine_settings=None):
|
|||
def request(query, params):
|
||||
offset = (params['pageno'] - 1) * 20
|
||||
|
||||
params['url'] = search_url.format(query=urlencode({'q': query}),
|
||||
offset=offset,
|
||||
client_id=guest_client_id)
|
||||
params['url'] = search_url.format(query=urlencode({'q': query}), offset=offset, client_id=guest_client_id)
|
||||
|
||||
return params
|
||||
|
||||
|
|
@ -98,11 +100,15 @@ def response(resp):
|
|||
embedded = embedded_url.format(uri=uri)
|
||||
|
||||
# append result
|
||||
results.append({'url': result['permalink_url'],
|
||||
results.append(
|
||||
{
|
||||
'url': result['permalink_url'],
|
||||
'title': title,
|
||||
'publishedDate': publishedDate,
|
||||
'embedded': embedded,
|
||||
'content': content})
|
||||
'content': content,
|
||||
}
|
||||
)
|
||||
|
||||
# return results
|
||||
return results
|
||||
|
|
|
|||
|
|
@ -42,9 +42,10 @@ def request(query, params):
|
|||
r = http_post(
|
||||
'https://accounts.spotify.com/api/token',
|
||||
data={'grant_type': 'client_credentials'},
|
||||
headers={'Authorization': 'Basic ' + base64.b64encode(
|
||||
"{}:{}".format(api_client_id, api_client_secret).encode()
|
||||
).decode()}
|
||||
headers={
|
||||
'Authorization': 'Basic '
|
||||
+ base64.b64encode("{}:{}".format(api_client_id, api_client_secret).encode()).decode()
|
||||
},
|
||||
)
|
||||
j = loads(r.text)
|
||||
params['headers'] = {'Authorization': 'Bearer {}'.format(j.get('access_token'))}
|
||||
|
|
@ -63,18 +64,12 @@ def response(resp):
|
|||
if result['type'] == 'track':
|
||||
title = result['name']
|
||||
url = result['external_urls']['spotify']
|
||||
content = '{} - {} - {}'.format(
|
||||
result['artists'][0]['name'],
|
||||
result['album']['name'],
|
||||
result['name'])
|
||||
content = '{} - {} - {}'.format(result['artists'][0]['name'], result['album']['name'], result['name'])
|
||||
|
||||
embedded = embedded_url.format(audioid=result['id'])
|
||||
|
||||
# append result
|
||||
results.append({'url': url,
|
||||
'title': title,
|
||||
'embedded': embedded,
|
||||
'content': content})
|
||||
results.append({'url': url, 'title': title, 'embedded': embedded, 'content': content})
|
||||
|
||||
# return results
|
||||
return results
|
||||
|
|
|
|||
|
|
@ -26,15 +26,11 @@ api_key = 'unset'
|
|||
|
||||
base_url = 'https://api.springernature.com/metadata/json?'
|
||||
|
||||
|
||||
def request(query, params):
|
||||
if api_key == 'unset':
|
||||
raise SearxEngineAPIException('missing Springer-Nature API key')
|
||||
args = urlencode({
|
||||
'q' : query,
|
||||
's' : nb_per_page * (params['pageno'] - 1),
|
||||
'p' : nb_per_page,
|
||||
'api_key' : api_key
|
||||
})
|
||||
args = urlencode({'q': query, 's': nb_per_page * (params['pageno'] - 1), 'p': nb_per_page, 'api_key': api_key})
|
||||
params['url'] = base_url + args
|
||||
logger.debug("query_url --> %s", params['url'])
|
||||
return params
|
||||
|
|
@ -50,21 +46,27 @@ def response(resp):
|
|||
content += "..."
|
||||
published = datetime.strptime(record['publicationDate'], '%Y-%m-%d')
|
||||
|
||||
metadata = [record[x] for x in [
|
||||
metadata = [
|
||||
record[x]
|
||||
for x in [
|
||||
'publicationName',
|
||||
'identifier',
|
||||
'contentType',
|
||||
] if record.get(x) is not None]
|
||||
]
|
||||
if record.get(x) is not None
|
||||
]
|
||||
|
||||
metadata = ' / '.join(metadata)
|
||||
if record.get('startingPage') and record.get('endingPage') is not None:
|
||||
metadata += " (%(startingPage)s-%(endingPage)s)" % record
|
||||
|
||||
results.append({
|
||||
results.append(
|
||||
{
|
||||
'title': record['title'],
|
||||
'url': record['url'][0]['value'].replace('http://', 'https://', 1),
|
||||
'content': content,
|
||||
'publishedDate': published,
|
||||
'metadata' : metadata
|
||||
})
|
||||
'metadata': metadata,
|
||||
}
|
||||
)
|
||||
return results
|
||||
|
|
|
|||
|
|
@ -49,7 +49,7 @@ def search(query, params):
|
|||
'query': query,
|
||||
'wildcard': r'%' + query.replace(' ', r'%') + r'%',
|
||||
'limit': limit,
|
||||
'offset': (params['pageno'] - 1) * limit
|
||||
'offset': (params['pageno'] - 1) * limit,
|
||||
}
|
||||
query_to_run = query_str + ' LIMIT :limit OFFSET :offset'
|
||||
|
||||
|
|
|
|||
|
|
@ -29,20 +29,24 @@ api_order = 'desc'
|
|||
# https://api.stackexchange.com/docs/advanced-search
|
||||
search_api = 'https://api.stackexchange.com/2.3/search/advanced?'
|
||||
|
||||
|
||||
def request(query, params):
|
||||
|
||||
args = urlencode({
|
||||
args = urlencode(
|
||||
{
|
||||
'q': query,
|
||||
'page': params['pageno'],
|
||||
'pagesize': pagesize,
|
||||
'site': api_site,
|
||||
'sort': api_sort,
|
||||
'order': 'desc',
|
||||
})
|
||||
}
|
||||
)
|
||||
params['url'] = search_api + args
|
||||
|
||||
return params
|
||||
|
||||
|
||||
def response(resp):
|
||||
|
||||
results = []
|
||||
|
|
@ -56,10 +60,12 @@ def response(resp):
|
|||
content += ' // is answered'
|
||||
content += " // score: %s" % result['score']
|
||||
|
||||
results.append({
|
||||
results.append(
|
||||
{
|
||||
'url': "https://%s.com/q/%s" % (api_site, result['question_id']),
|
||||
'title': html.unescape(result['title']),
|
||||
'content': html.unescape(content),
|
||||
})
|
||||
}
|
||||
)
|
||||
|
||||
return results
|
||||
|
|
|
|||
|
|
@ -123,15 +123,10 @@ def response(resp):
|
|||
|
||||
if published_date:
|
||||
# append result
|
||||
results.append({'url': url,
|
||||
'title': title,
|
||||
'content': content,
|
||||
'publishedDate': published_date})
|
||||
results.append({'url': url, 'title': title, 'content': content, 'publishedDate': published_date})
|
||||
else:
|
||||
# append result
|
||||
results.append({'url': url,
|
||||
'title': title,
|
||||
'content': content})
|
||||
results.append({'url': url, 'title': title, 'content': content})
|
||||
|
||||
# return results
|
||||
return results
|
||||
|
|
@ -152,7 +147,7 @@ def _fetch_supported_languages(resp):
|
|||
'malayam': 'ml',
|
||||
'norsk': 'nb',
|
||||
'sinhalese': 'si',
|
||||
'sudanese': 'su'
|
||||
'sudanese': 'su',
|
||||
}
|
||||
|
||||
# get the English name of every language known by babel
|
||||
|
|
|
|||
|
|
@ -56,11 +56,7 @@ def response(resp):
|
|||
name_row = rows[i]
|
||||
|
||||
links = name_row.xpath('./td[@class="desc-top"]/a')
|
||||
params = {
|
||||
'template': 'torrent.html',
|
||||
'url': links[-1].attrib.get('href'),
|
||||
'title': extract_text(links[-1])
|
||||
}
|
||||
params = {'template': 'torrent.html', 'url': links[-1].attrib.get('href'), 'title': extract_text(links[-1])}
|
||||
# I have not yet seen any torrents without magnet links, but
|
||||
# it's better to be prepared to stumble upon one some day
|
||||
if len(links) == 2:
|
||||
|
|
|
|||
|
|
@ -35,10 +35,12 @@ api_key = ''
|
|||
# https://newznab.readthedocs.io/en/latest/misc/api/#predefined-categories
|
||||
torznab_categories = []
|
||||
|
||||
|
||||
def init(engine_settings=None): # pylint: disable=unused-argument
|
||||
if len(base_url) < 1:
|
||||
raise ValueError('missing torznab base_url')
|
||||
|
||||
|
||||
def request(query, params):
|
||||
|
||||
search_url = base_url + '?t=search&q={search_query}'
|
||||
|
|
@ -48,13 +50,12 @@ def request(query, params):
|
|||
search_url += '&cat={torznab_categories}'
|
||||
|
||||
params['url'] = search_url.format(
|
||||
search_query = quote(query),
|
||||
api_key = api_key,
|
||||
torznab_categories = ",".join([str(x) for x in torznab_categories])
|
||||
search_query=quote(query), api_key=api_key, torznab_categories=",".join([str(x) for x in torznab_categories])
|
||||
)
|
||||
|
||||
return params
|
||||
|
||||
|
||||
def response(resp):
|
||||
results = []
|
||||
|
||||
|
|
@ -103,8 +104,7 @@ def response(resp):
|
|||
|
||||
result["publishedDate"] = None
|
||||
try:
|
||||
result["publishedDate"] = datetime.strptime(
|
||||
get_property(item, 'pubDate'), '%a, %d %b %Y %H:%M:%S %z')
|
||||
result["publishedDate"] = datetime.strptime(get_property(item, 'pubDate'), '%a, %d %b %Y %H:%M:%S %z')
|
||||
except (ValueError, TypeError) as e:
|
||||
logger.debug("ignore exception (publishedDate): %s", e)
|
||||
|
||||
|
|
@ -134,9 +134,7 @@ def get_property(item, property_name):
|
|||
def get_torznab_attr(item, attr_name):
|
||||
element = item.find(
|
||||
'.//torznab:attr[@name="{attr_name}"]'.format(attr_name=attr_name),
|
||||
{
|
||||
'torznab': 'http://torznab.com/schemas/2015/feed'
|
||||
}
|
||||
{'torznab': 'http://torznab.com/schemas/2015/feed'},
|
||||
)
|
||||
|
||||
if element is not None:
|
||||
|
|
|
|||
|
|
@ -28,24 +28,25 @@ def request(query, params):
|
|||
key_form = '&key=' + api_key
|
||||
else:
|
||||
key_form = ''
|
||||
params['url'] = url.format(from_lang=params['from_lang'][1],
|
||||
to_lang=params['to_lang'][1],
|
||||
query=params['query'],
|
||||
key=key_form)
|
||||
params['url'] = url.format(
|
||||
from_lang=params['from_lang'][1], to_lang=params['to_lang'][1], query=params['query'], key=key_form
|
||||
)
|
||||
return params
|
||||
|
||||
|
||||
def response(resp):
|
||||
results = []
|
||||
results.append({
|
||||
results.append(
|
||||
{
|
||||
'url': web_url.format(
|
||||
from_lang=resp.search_params['from_lang'][2],
|
||||
to_lang=resp.search_params['to_lang'][2],
|
||||
query=resp.search_params['query']),
|
||||
query=resp.search_params['query'],
|
||||
),
|
||||
'title': '[{0}-{1}] {2}'.format(
|
||||
resp.search_params['from_lang'][1],
|
||||
resp.search_params['to_lang'][1],
|
||||
resp.search_params['query']),
|
||||
'content': resp.json()['responseData']['translatedText']
|
||||
})
|
||||
resp.search_params['from_lang'][1], resp.search_params['to_lang'][1], resp.search_params['query']
|
||||
),
|
||||
'content': resp.json()['responseData']['translatedText'],
|
||||
}
|
||||
)
|
||||
return results
|
||||
|
|
|
|||
|
|
@ -26,23 +26,13 @@ paging = True
|
|||
|
||||
def clean_url(url):
|
||||
parsed = urlparse(url)
|
||||
query = [(k, v) for (k, v)
|
||||
in parse_qsl(parsed.query) if k not in ['ixid', 's']]
|
||||
query = [(k, v) for (k, v) in parse_qsl(parsed.query) if k not in ['ixid', 's']]
|
||||
|
||||
return urlunparse((
|
||||
parsed.scheme,
|
||||
parsed.netloc,
|
||||
parsed.path,
|
||||
parsed.params,
|
||||
urlencode(query),
|
||||
parsed.fragment
|
||||
))
|
||||
return urlunparse((parsed.scheme, parsed.netloc, parsed.path, parsed.params, urlencode(query), parsed.fragment))
|
||||
|
||||
|
||||
def request(query, params):
|
||||
params['url'] = search_url + urlencode({
|
||||
'query': query, 'page': params['pageno'], 'per_page': page_size
|
||||
})
|
||||
params['url'] = search_url + urlencode({'query': query, 'page': params['pageno'], 'per_page': page_size})
|
||||
logger.debug("query_url --> %s", params['url'])
|
||||
return params
|
||||
|
||||
|
|
@ -53,13 +43,15 @@ def response(resp):
|
|||
|
||||
if 'results' in json_data:
|
||||
for result in json_data['results']:
|
||||
results.append({
|
||||
results.append(
|
||||
{
|
||||
'template': 'images.html',
|
||||
'url': clean_url(result['links']['html']),
|
||||
'thumbnail_src': clean_url(result['urls']['thumb']),
|
||||
'img_src': clean_url(result['urls']['raw']),
|
||||
'title': result.get('alt_description') or 'unknown',
|
||||
'content': result.get('description') or ''
|
||||
})
|
||||
'content': result.get('description') or '',
|
||||
}
|
||||
)
|
||||
|
||||
return results
|
||||
|
|
|
|||
|
|
@ -25,15 +25,16 @@ paging = True
|
|||
base_url = 'https://vimeo.com/'
|
||||
search_url = base_url + '/search/page:{pageno}?{query}'
|
||||
|
||||
embedded_url = '<iframe data-src="https://player.vimeo.com/video/{videoid}" ' +\
|
||||
'width="540" height="304" frameborder="0" ' +\
|
||||
'webkitallowfullscreen mozallowfullscreen allowfullscreen></iframe>'
|
||||
embedded_url = (
|
||||
'<iframe data-src="https://player.vimeo.com/video/{videoid}" '
|
||||
+ 'width="540" height="304" frameborder="0" '
|
||||
+ 'webkitallowfullscreen mozallowfullscreen allowfullscreen></iframe>'
|
||||
)
|
||||
|
||||
|
||||
# do search-request
|
||||
def request(query, params):
|
||||
params['url'] = search_url.format(pageno=params['pageno'],
|
||||
query=urlencode({'q': query}))
|
||||
params['url'] = search_url.format(pageno=params['pageno'], query=urlencode({'q': query}))
|
||||
|
||||
return params
|
||||
|
||||
|
|
@ -56,13 +57,17 @@ def response(resp):
|
|||
embedded = embedded_url.format(videoid=videoid)
|
||||
|
||||
# append result
|
||||
results.append({'url': url,
|
||||
results.append(
|
||||
{
|
||||
'url': url,
|
||||
'title': title,
|
||||
'content': '',
|
||||
'template': 'videos.html',
|
||||
'publishedDate': publishedDate,
|
||||
'embedded': embedded,
|
||||
'thumbnail': thumbnail})
|
||||
'thumbnail': thumbnail,
|
||||
}
|
||||
)
|
||||
|
||||
# return results
|
||||
return results
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue