forked from Ponysearch/Ponysearch
Merge branch 'master' into flask_perimeter
This commit is contained in:
Adam Tauber
GitHub
commit
627962ce40
13 changed files with 179 additions and 147 deletions
32
manage.sh
32
manage.sh
|
|
@ -14,6 +14,36 @@ update_dev_packages() {
|
|||
pip install --upgrade -r "$BASE_DIR/requirements-dev.txt"
|
||||
}
|
||||
|
||||
check_geckodriver() {
|
||||
echo '[!] Checking geckodriver'
|
||||
set -e
|
||||
geckodriver -V 2>1 > /dev/null || NOTFOUND=1
|
||||
set +e
|
||||
if [ -z $NOTFOUND ]; then
|
||||
return
|
||||
fi
|
||||
GECKODRIVER_VERSION="v0.11.1"
|
||||
PLATFORM=`python -c "import platform; print platform.system().lower(), platform.architecture()[0]"`
|
||||
case $PLATFORM in
|
||||
"linux 32bit" | "linux2 32bit") ARCH="linux32";;
|
||||
"linux 64bit" | "linux2 64bit") ARCH="linux64";;
|
||||
"windows 32 bit") ARCH="win32";;
|
||||
"windows 64 bit") ARCH="win64";;
|
||||
"mac 64bit") ARCH="macos";;
|
||||
esac
|
||||
GECKODRIVER_URL="https://github.com/mozilla/geckodriver/releases/download/$GECKODRIVER_VERSION/geckodriver-$GECKODRIVER_VERSION-$ARCH.tar.gz";
|
||||
if [ -z "$VIRTUAL_ENV" ]; then
|
||||
echo "geckodriver can't be installed because VIRTUAL_ENV is not set, you should download it from\n $GECKODRIVER_URL"
|
||||
exit
|
||||
else
|
||||
echo "Installing $VIRTUAL_ENV from\n $GECKODRIVER_URL"
|
||||
FILE=`mktemp`
|
||||
wget "$GECKODRIVER_URL" -qO $FILE && tar xz -C $VIRTUAL_ENV/bin/ -f $FILE geckodriver
|
||||
rm $FILE
|
||||
chmod 777 $VIRTUAL_ENV/bin/geckodriver
|
||||
fi
|
||||
}
|
||||
|
||||
pep8_check() {
|
||||
echo '[!] Running pep8 check'
|
||||
# ignored rules:
|
||||
|
|
@ -43,6 +73,7 @@ tests() {
|
|||
set -e
|
||||
pep8_check
|
||||
unit_tests
|
||||
check_geckodriver
|
||||
robot_tests
|
||||
set +e
|
||||
}
|
||||
|
|
@ -88,6 +119,7 @@ Commands
|
|||
unit_tests - Run unit tests
|
||||
update_dev_packages - Check & update development and production dependency changes
|
||||
update_packages - Check & update dependency changes
|
||||
check_geckodriver - Check & download geckodriver (required for robot_tests)
|
||||
"
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,12 +1,12 @@
|
|||
certifi==2016.9.26
|
||||
flask==0.11.1
|
||||
flask==0.12
|
||||
flask-babel==0.11.1
|
||||
lxml==3.6.0
|
||||
ndg-httpsclient==0.4.1
|
||||
lxml==3.7.1
|
||||
ndg-httpsclient==0.4.2
|
||||
pyasn1==0.1.9
|
||||
pyasn1-modules==0.0.8
|
||||
pygments==2.1.3
|
||||
pyopenssl==0.15.1
|
||||
pyopenssl==16.2.0
|
||||
python-dateutil==2.5.3
|
||||
pyyaml==3.11
|
||||
requests[socks]==2.10.0
|
||||
requests[socks]==2.12.4
|
||||
|
|
|
|||
|
|
@ -81,22 +81,22 @@ def searx_bang(full_query):
|
|||
engine_query = full_query.getSearchQuery()[1:]
|
||||
|
||||
for lc in language_codes:
|
||||
lang_id, lang_name, country, english_name = map(str.lower, lc)
|
||||
lang_id, lang_name, country, english_name = map(unicode.lower, lc)
|
||||
|
||||
# check if query starts with language-id
|
||||
if lang_id.startswith(engine_query):
|
||||
if len(engine_query) <= 2:
|
||||
results.append(':{lang_id}'.format(lang_id=lang_id.split('-')[0]))
|
||||
results.append(u':{lang_id}'.format(lang_id=lang_id.split('-')[0]))
|
||||
else:
|
||||
results.append(':{lang_id}'.format(lang_id=lang_id))
|
||||
results.append(u':{lang_id}'.format(lang_id=lang_id))
|
||||
|
||||
# check if query starts with language name
|
||||
if lang_name.startswith(engine_query) or english_name.startswith(engine_query):
|
||||
results.append(':{lang_name}'.format(lang_name=lang_name))
|
||||
results.append(u':{lang_name}'.format(lang_name=lang_name))
|
||||
|
||||
# check if query starts with country
|
||||
if country.startswith(engine_query.replace('_', ' ')):
|
||||
results.append(':{country}'.format(country=country.replace(' ', '_')))
|
||||
results.append(u':{country}'.format(country=country.replace(' ', '_')))
|
||||
|
||||
# remove duplicates
|
||||
result_set = set(results)
|
||||
|
|
|
|||
|
|
@ -42,7 +42,9 @@ def extract_text(xpath_results):
|
|||
return ''.join(xpath_results)
|
||||
else:
|
||||
# it's a element
|
||||
return html_to_text(xpath_results.text_content()).strip()
|
||||
text = html.tostring(xpath_results, encoding='unicode', method='text', with_tail=False)
|
||||
text = text.strip().replace('\n', ' ')
|
||||
return ' '.join(text.split())
|
||||
|
||||
|
||||
def extract_url(xpath_results, search_url):
|
||||
|
|
|
|||
|
|
@ -4,13 +4,9 @@ $(document).ready(function() {
|
|||
if ($(document).height() - win.height() == win.scrollTop()) {
|
||||
var formData = $('#pagination form:last').serialize();
|
||||
if (formData) {
|
||||
var pageno = $('#pagination input[name=pageno]:last').attr('value');
|
||||
$('#pagination').html('<div class="loading-spinner"></div>');
|
||||
$.post('./', formData, function (data) {
|
||||
var lastImageHref = $('.result-images:last a').attr('href');
|
||||
var body = $(data);
|
||||
$('a[href^="#open-modal"]:last').attr('href', '#open-modal-1-' + pageno);
|
||||
body.find('.modal-image a:first').attr('href', lastImageHref);
|
||||
$('#pagination').remove();
|
||||
$('#main_results').append('<hr/>');
|
||||
$('#main_results').append(body.find('.result'));
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -1,77 +0,0 @@
|
|||
.modal-image {
|
||||
position: fixed;
|
||||
top: 0;
|
||||
right: 0;
|
||||
bottom: 0;
|
||||
left: 0;
|
||||
background: rgba(0,0,0,0.8);
|
||||
z-index: 1000000001;
|
||||
opacity:0 !important;
|
||||
pointer-events: none;
|
||||
|
||||
button {
|
||||
display: none;
|
||||
}
|
||||
|
||||
&:target {
|
||||
opacity: 1 !important;
|
||||
pointer-events: auto;
|
||||
}
|
||||
|
||||
& > div {
|
||||
margin: 2% auto;
|
||||
width: 97%;
|
||||
background: @dim-gray;
|
||||
border: @gray 0.1rem solid;
|
||||
}
|
||||
|
||||
@media (min-width: 769px) {
|
||||
& > div {
|
||||
max-width: 60.0rem;
|
||||
}
|
||||
}
|
||||
|
||||
.image-paging-left {
|
||||
margin-right: 1.0rem;
|
||||
margin-top: 0.5rem;
|
||||
width:15px;
|
||||
height:15px;
|
||||
background: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAA8AAAAPCAYAAAA71pVKAAAABmJLR0QA/wD/AP+gvaeTAAAACXBI
|
||||
WXMAAAsTAAALEwEAmpwYAAAAB3RJTUUH4AweDQoOuikqUQAAAB1pVFh0Q29tbWVudAAAAAAAQ3Jl
|
||||
YXRlZCB3aXRoIEdJTVBkLmUHAAAAiElEQVQoz6XTrQ0CQRCG4SesQHI5g6EAqIEewNLSVUACzfAT
|
||||
BApDDSgSBAaJORKyauf2czOZdybzl5SpxR5j3H/OUQHYoMMMNwE1fcUT5hFwUgPuenAxBDxHwRZb
|
||||
HKMgbPDCuiQ4ZfYDU6xwxTNafXDP1dOu3nP1heUJDnmCVAB/cMES7/+v+gIq0Bs3k6NL9AAAAABJ
|
||||
RU5ErkJggg==) 96% no-repeat;
|
||||
}
|
||||
|
||||
.image-paging-right {
|
||||
margin-left: 1.2rem;
|
||||
margin-top: 0.5rem;
|
||||
width:15px;
|
||||
height:15px;
|
||||
background: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAA8AAAAPCAYAAAA71pVKAAAABmJLR0QA/wD/AP+gvaeTAAAACXBI
|
||||
WXMAAAsTAAALEwEAmpwYAAAAB3RJTUUH4AweDQon+JuyPQAAAB1pVFh0Q29tbWVudAAAAAAAQ3Jl
|
||||
YXRlZCB3aXRoIEdJTVBkLmUHAAAAaklEQVQoz73TsQ2DUAxF0SMWAFEzwGcaWhgpEyAlbQYJMACj
|
||||
sAINTaiIf8Tt3FzL9jPfDHijdoHiVK9o8EAlQMKM1z8EZUTQHoJnjmDJFUwYr17hTIcN/W2dwzOH
|
||||
tx2+czhhCZ9oNH/6qh1F2RaYgWxrQwAAAABJRU5ErkJggg==);
|
||||
|
||||
}
|
||||
|
||||
.image-container::before {
|
||||
display: block;
|
||||
min-width: 1.0rem;
|
||||
max-width: 60.0rem;
|
||||
min-height: 10.0rem;
|
||||
height: 30.0rem;
|
||||
content: "";
|
||||
}
|
||||
}
|
||||
|
||||
.modal-close {
|
||||
position:fixed;
|
||||
top: 0;
|
||||
left: 0;
|
||||
height: 100% !important;
|
||||
width: 100% !important;
|
||||
z-index: -1;
|
||||
}
|
||||
|
|
@ -19,5 +19,3 @@
|
|||
@import "cursor.less";
|
||||
|
||||
@import "code.less";
|
||||
|
||||
@import "modal-pic.less";
|
||||
|
|
|
|||
|
|
@ -17,7 +17,3 @@
|
|||
@import "code.less";
|
||||
|
||||
@import "navbar.less";
|
||||
|
||||
@import "../logicodev/variables.less";
|
||||
|
||||
@import "../logicodev/modal-pic.less";
|
||||
|
|
|
|||
|
|
@ -1,54 +1,39 @@
|
|||
<a href="#open-modal-{{ index }}-{{ pageno }}">
|
||||
<img src="{% if result.thumbnail_src %}{{ image_proxify(result.thumbnail_src) }}{% else %}{{ image_proxify(result.img_src) }}{% endif %}" alt="{{ result.title|striptags }}" title="{{ result.title|striptags }}" class="img-thumbnail" id="img-result-thumb-{{ index }}" />
|
||||
{% from 'oscar/macros.html' import draw_favicon %}
|
||||
|
||||
<a href="{{ result.img_src }}" {% if results_on_new_tab %}target="_blank" rel="noopener noreferrer"{% else %}rel="noreferrer"{% endif %} data-toggle="modal" data-target="#modal-{{ index }}-{{pageno}}">
|
||||
<img src="{% if result.thumbnail_src %}{{ image_proxify(result.thumbnail_src) }}{% else %}{{ image_proxify(result.img_src) }}{% endif %}" alt="{{ result.title|striptags }}" title="{{ result.title|striptags }}" class="img-thumbnail">
|
||||
</a>
|
||||
|
||||
<style type="text/css" media="screen">
|
||||
#open-modal-{{ index }}-{{ pageno }}:target .image-container::before {
|
||||
background: url({{ image_proxify(result.img_src)|safe }}) no-repeat center/contain;
|
||||
}
|
||||
</style>
|
||||
|
||||
<div id="open-modal-{{ index }}-{{ pageno }}" class="modal-image">
|
||||
<div class="container modal-dialog">
|
||||
<div class="row">
|
||||
<div class="col-md-12 col-sm-12 col-xs-12 modal-header">
|
||||
<a {% if index != 1 %}href="#open-modal-{{ index-1 }}-{{ pageno }}"{% endif %}>
|
||||
<span class="pull-left image-paging-left"></span>
|
||||
</a>
|
||||
<a href="#open-modal-{{ index+1 }}-{{ pageno }}">
|
||||
<span class="pull-right image-paging-right"></span>
|
||||
</a>
|
||||
<h4 class="modal-title image-title">{{ result.title|striptags }}</h4>
|
||||
<div class="modal fade" id="modal-{{ index }}-{{ pageno }}" tabindex="-1" role="dialog" aria-hidden="true">
|
||||
<div class="modal-dialog">
|
||||
<div class="modal-wrapper">
|
||||
<div class="modal-header">
|
||||
<button type="button" class="close" data-dismiss="modal"><span aria-hidden="true">×</span><span class="sr-only">Close</span></button>
|
||||
<h4 class="modal-title">{% if result.engine~".png" in favicons %}{{ draw_favicon(result.engine) }} {% endif %}{{ result.title|striptags }}</h4>
|
||||
</div>
|
||||
</div>
|
||||
<div class="row">
|
||||
<div class="col-md-12 col-sm-12 col-xs-12 modal-body">
|
||||
<a href="{{ result.img_src }}" {% if results_on_new_tab %}target="_blank" rel="noopener noreferrer"{% else %}rel="noreferrer"{% endif %}><div class="image-container"></div></a>
|
||||
</div>
|
||||
</div>
|
||||
{% if result.content %}
|
||||
<div class="row">
|
||||
<div class="col-md-12 hidden-sm hidden-xs modal-body">
|
||||
<p class="result-content">{{ result.content|safe }}</p>
|
||||
<div class="modal-body">
|
||||
<img class="img-responsive center-block" src="{% if result.thumbnail_src %}{{ image_proxify(result.thumbnail_src) }}{% else %}{{ image_proxify(result.img_src) }}{% endif %}" alt="{{ result.title|striptags }}">
|
||||
{% if result.author %}<span class="photo-author">{{ result.author }}</span><br />{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
{% if result.content %}
|
||||
<p class="result-content">
|
||||
{{ result.content }}
|
||||
</p>
|
||||
{% endif %}
|
||||
</div>
|
||||
<div class="modal-footer">
|
||||
<div class="row">
|
||||
<div class="col-md-10 col-xs-12">
|
||||
<p class="text-muted pull-left">{{ result.pretty_url }}</p>
|
||||
</div>
|
||||
<div class="col-md-2 hidden-sm hidden-xs">
|
||||
<div class="clearfix"></div>
|
||||
<span class="label label-default pull-right">{{ result.engine }}</span>
|
||||
</div>
|
||||
</div>
|
||||
<p class="text-muted pull-left">{{ result.pretty_url }}</p>
|
||||
<div class="clearfix"></div>
|
||||
<div class="row">
|
||||
<a href="{{ result.url }}" class="btn btn-default">
|
||||
{{ _('View source') }}
|
||||
</a>
|
||||
<div class="col-md-6">
|
||||
<a href="{{ result.img_src }}" {% if results_on_new_tab %}target="_blank" rel="noopener noreferrer"{% else %}rel="noreferrer"{% endif %} class="btn btn-default">{{ _('Get image') }}</a>
|
||||
</div>
|
||||
<div class="col-md-6">
|
||||
<a href="{{ result.url }}" {% if results_on_new_tab %}target="_blank" rel="noopener noreferrer"{% else %}rel="noreferrer"{% endif %} class="btn btn-default">{{ _('View source') }}</a>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<a href="#img-result-thumb-{{ index }}-{{ pageno }}" class="modal-close"></a>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -13,10 +13,9 @@ request_params = default_request_params()
|
|||
# Possible params
|
||||
# request_params['headers']['User-Agent'] = ''
|
||||
# request_params['category'] = ''
|
||||
# request_params['started'] = ''
|
||||
|
||||
request_params['pageno'] = 1
|
||||
request_params['language'] = 'en_us'
|
||||
request_params['time_range'] = ''
|
||||
|
||||
params = google.request(argv[1], request_params)
|
||||
|
||||
|
|
@ -32,5 +31,5 @@ else:
|
|||
request_args['data'] = request_params['data']
|
||||
|
||||
resp = req(request_params['url'], **request_args)
|
||||
|
||||
resp.search_params = request_params
|
||||
print(dumps(google.response(resp)))
|
||||
101
utils/standalone_searx.py
Executable file
101
utils/standalone_searx.py
Executable file
|
|
@ -0,0 +1,101 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
'''
|
||||
searx is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
searx is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with searx. If not, see < http://www.gnu.org/licenses/ >.
|
||||
|
||||
(C) 2016- by Alexandre Flament, <alex@al-f.net>
|
||||
'''
|
||||
|
||||
# set path
|
||||
from sys import path
|
||||
from os.path import realpath, dirname
|
||||
path.append(realpath(dirname(realpath(__file__)) + '/../'))
|
||||
|
||||
# initialization
|
||||
from json import dumps
|
||||
from searx import settings
|
||||
import searx.query
|
||||
import searx.search
|
||||
import searx.engines
|
||||
import searx.preferences
|
||||
import argparse
|
||||
|
||||
searx.engines.initialize_engines(settings['engines'])
|
||||
|
||||
# command line parsing
|
||||
parser = argparse.ArgumentParser(description='Standalone searx.')
|
||||
parser.add_argument('query', type=str,
|
||||
help='Text query')
|
||||
parser.add_argument('--category', type=str, nargs='?',
|
||||
choices=searx.engines.categories.keys(),
|
||||
default='general',
|
||||
help='Search category')
|
||||
parser.add_argument('--lang', type=str, nargs='?',default='all',
|
||||
help='Search language')
|
||||
parser.add_argument('--pageno', type=int, nargs='?', default=1,
|
||||
help='Page number starting from 1')
|
||||
parser.add_argument('--safesearch', type=str, nargs='?', choices=['0', '1', '2'], default='0',
|
||||
help='Safe content filter from none to strict')
|
||||
parser.add_argument('--timerange', type=str, nargs='?', choices=['day', 'week', 'month', 'year'],
|
||||
help='Filter by time range')
|
||||
args = parser.parse_args()
|
||||
|
||||
# search results for the query
|
||||
form = {
|
||||
"q":args.query,
|
||||
"categories":args.category.decode('utf-8'),
|
||||
"pageno":str(args.pageno),
|
||||
"language":args.lang,
|
||||
"time_range":args.timerange
|
||||
}
|
||||
preferences = searx.preferences.Preferences(['oscar'], searx.engines.categories.keys(), searx.engines.engines, [])
|
||||
preferences.key_value_settings['safesearch'].parse(args.safesearch)
|
||||
|
||||
search_query = searx.search.get_search_query_from_webapp(preferences, form)
|
||||
search = searx.search.Search(search_query)
|
||||
result_container = search.search()
|
||||
|
||||
# output
|
||||
from datetime import datetime
|
||||
|
||||
def no_parsed_url(results):
|
||||
for result in results:
|
||||
del result['parsed_url']
|
||||
return results
|
||||
|
||||
def json_serial(obj):
|
||||
"""JSON serializer for objects not serializable by default json code"""
|
||||
if isinstance(obj, datetime):
|
||||
serial = obj.isoformat()
|
||||
return serial
|
||||
raise TypeError ("Type not serializable")
|
||||
|
||||
result_container_json = {
|
||||
"search": {
|
||||
"q": search_query.query,
|
||||
"pageno": search_query.pageno,
|
||||
"lang": search_query.lang,
|
||||
"safesearch": search_query.safesearch,
|
||||
"timerange": search_query.time_range,
|
||||
"engines": search_query.engines
|
||||
},
|
||||
"results": no_parsed_url(result_container.get_ordered_results()),
|
||||
"infoboxes": result_container.infoboxes,
|
||||
"suggestions": list(result_container.suggestions),
|
||||
"answers": list(result_container.answers),
|
||||
"paging": result_container.paging,
|
||||
"results_number": result_container.results_number()
|
||||
}
|
||||
|
||||
print(dumps(result_container_json, sort_keys=True, indent=4, ensure_ascii=False, encoding="utf-8", default=json_serial))
|
||||
Loading…
Reference in a new issue