From 9299355570e32c4d24d7274d716eca1a93119d13 Mon Sep 17 00:00:00 2001 From: Marc Abonce Seguin Date: Sun, 24 Nov 2019 20:21:37 -0700 Subject: [PATCH] add seedpeer again --- searx/engines/seedpeer.py | 78 +++++++++++++++++++ searx/settings.yml | 5 ++ .../courgette/result_templates/torrent.html | 2 +- .../legacy/result_templates/torrent.html | 2 +- .../oscar/result_templates/torrent.html | 2 +- .../simple/result_templates/torrent.html | 2 +- tests/unit/engines/test_seedpeer.py | 66 ++++++++++++++++ 7 files changed, 153 insertions(+), 4 deletions(-) create mode 100644 searx/engines/seedpeer.py create mode 100644 tests/unit/engines/test_seedpeer.py diff --git a/searx/engines/seedpeer.py b/searx/engines/seedpeer.py new file mode 100644 index 000000000..f9b1f99c8 --- /dev/null +++ b/searx/engines/seedpeer.py @@ -0,0 +1,78 @@ +# Seedpeer (Videos, Music, Files) +# +# @website https://seedpeer.me +# @provide-api no (nothing found) +# +# @using-api no +# @results HTML (using search portal) +# @stable yes (HTML can change) +# @parse url, title, content, seed, leech, magnetlink + +from lxml import html +from json import loads +from operator import itemgetter +from searx.url_utils import quote, urljoin +from searx.engines.xpath import extract_text + + +url = 'https://seedpeer.me/' +search_url = url + 'search/{search_term}?page={page_no}' +torrent_file_url = url + 'torrent/{torrent_hash}' + +# specific xpath variables +script_xpath = '//script[@type="text/javascript"][not(@src)]' +torrent_xpath = '(//table)[2]/tbody/tr' +link_xpath = '(./td)[1]/a/@href' +age_xpath = '(./td)[2]' +size_xpath = '(./td)[3]' + + +# do search-request +def request(query, params): + params['url'] = search_url.format(search_term=quote(query), + page_no=params['pageno']) + return params + + +# get response from search-request +def response(resp): + results = [] + dom = html.fromstring(resp.text) + result_rows = dom.xpath(torrent_xpath) + + try: + script_element = dom.xpath(script_xpath)[0] + json_string = script_element.text[script_element.text.find('{'):] + torrents_json = loads(json_string) + except: + return [] + + # parse results + for torrent_row, torrent_json in zip(result_rows, torrents_json['data']['list']): + title = torrent_json['name'] + seed = int(torrent_json['seeds']) + leech = int(torrent_json['peers']) + size = int(torrent_json['size']) + torrent_hash = torrent_json['hash'] + + torrentfile = torrent_file_url.format(torrent_hash=torrent_hash) + magnetlink = 'magnet:?xt=urn:btih:{}'.format(torrent_hash) + + age = extract_text(torrent_row.xpath(age_xpath)) + link = torrent_row.xpath(link_xpath)[0] + + href = urljoin(url, link) + + # append result + results.append({'url': href, + 'title': title, + 'content': age, + 'seed': seed, + 'leech': leech, + 'filesize': size, + 'torrentfile': torrentfile, + 'magnetlink': magnetlink, + 'template': 'torrent.html'}) + + # return results sorted by seeder + return sorted(results, key=itemgetter('seed'), reverse=True) diff --git a/searx/settings.yml b/searx/settings.yml index 835fbe5f6..25d90d4db 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -748,6 +748,11 @@ engines: page_size : 10 disabled : True + - name : seedpeer + shortcut : speu + engine : seedpeer + categories: files, music, videos + # - name : yacy # engine : yacy # shortcut : ya diff --git a/searx/templates/courgette/result_templates/torrent.html b/searx/templates/courgette/result_templates/torrent.html index d659064d9..7f94a221e 100644 --- a/searx/templates/courgette/result_templates/torrent.html +++ b/searx/templates/courgette/result_templates/torrent.html @@ -4,7 +4,7 @@ {% endif %}

{{ result.title|safe }}

{% if result.content %}{{ result.content|safe }}
{% endif %} - {% if result.seed %}{{ _('Seeder') }} : {{ result.seed }}, {{ _('Leecher') }} : {{ result.leech }}
{% endif %} + {% if result.seed is defined %}{{ _('Seeder') }} : {{ result.seed }}, {{ _('Leecher') }} : {{ result.leech }}
{% endif %} {% if result.magnetlink %}{{ _('magnet link') }}{% endif %} {% if result.torrentfile %}{{ _('torrent file') }}{% endif %} diff --git a/searx/templates/legacy/result_templates/torrent.html b/searx/templates/legacy/result_templates/torrent.html index 7a8ac33de..068e05373 100644 --- a/searx/templates/legacy/result_templates/torrent.html +++ b/searx/templates/legacy/result_templates/torrent.html @@ -8,6 +8,6 @@

{% if result.magnetlink %}{{ _('magnet link') }}{% endif %} {% if result.torrentfile %}{{ _('torrent file') }}{% endif %} - - {% if result.seed %}{{ _('Seeder') }} : {{ result.seed }}, {{ _('Leecher') }} : {{ result.leech }}{% endif %} + {% if result.seed is defined %}{{ _('Seeder') }} : {{ result.seed }}, {{ _('Leecher') }} : {{ result.leech }}{% endif %}

diff --git a/searx/templates/oscar/result_templates/torrent.html b/searx/templates/oscar/result_templates/torrent.html index f5ea415e2..089367e36 100644 --- a/searx/templates/oscar/result_templates/torrent.html +++ b/searx/templates/oscar/result_templates/torrent.html @@ -3,7 +3,7 @@ {{ result_header(result, favicons) }} {{ result_sub_header(result) }} -{% if result.seed %}

{{ icon('transfer') }} {{ _('Seeder') }} {{ result.seed }} • {{ _('Leecher') }} {{ result.leech }}{% endif %} +{% if result.seed is defined %}

{{ icon('transfer') }} {{ _('Seeder') }} {{ result.seed }} • {{ _('Leecher') }} {{ result.leech }}{% endif %} {% if result.filesize %}
{{ icon('floppy-disk') }} {{ _('Filesize') }} {% if result.filesize < 1024 %}{{ result.filesize }} {{ _('Bytes') }} diff --git a/searx/templates/simple/result_templates/torrent.html b/searx/templates/simple/result_templates/torrent.html index 3c7fd15e8..71c775bc9 100644 --- a/searx/templates/simple/result_templates/torrent.html +++ b/searx/templates/simple/result_templates/torrent.html @@ -6,7 +6,7 @@ {% if result.magnetlink %}

• {{ result_link(result.magnetlink, icon('magnet') + _('magnet link'), "magnetlink") }}

{% endif %} {% if result.torrentfile %}

• {{ result_link(result.torrentfile, icon('download-alt') + _('torrent file'), "torrentfile") }}

{% endif %} -{% if result.seed %}

• {{ icon('arrow-swap') }} {{ _('Seeder') }} {{ result.seed }} • {{ _('Leecher') }} {{ result.leech }}

{% endif %} +{% if result.seed is defined %}

• {{ icon('arrow-swap') }} {{ _('Seeder') }} {{ result.seed }} • {{ _('Leecher') }} {{ result.leech }}

{% endif %} {%- if result.filesize %}

{{ icon('floppy-disk') }} {{ _('Filesize') }} {%- if result.filesize < 1024 %}{{ result.filesize }} {{ _('Bytes') }} diff --git a/tests/unit/engines/test_seedpeer.py b/tests/unit/engines/test_seedpeer.py new file mode 100644 index 000000000..2057c1cb1 --- /dev/null +++ b/tests/unit/engines/test_seedpeer.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- +from collections import defaultdict +import mock +from searx.engines import seedpeer +from searx.testing import SearxTestCase + + +class TestBtdiggEngine(SearxTestCase): + + def test_request(self): + query = 'test_query' + dicto = defaultdict(dict) + dicto['pageno'] = 1 + params = seedpeer.request(query, dicto) + self.assertIn('url', params) + self.assertIn(query, params['url']) + self.assertIn('seedpeer', params['url']) + + def test_response(self): + self.assertRaises(AttributeError, seedpeer.response, None) + self.assertRaises(AttributeError, seedpeer.response, []) + self.assertRaises(AttributeError, seedpeer.response, '') + self.assertRaises(AttributeError, seedpeer.response, '[]') + + response = mock.Mock(text='') + self.assertEqual(seedpeer.response(response), []) + + html = u""" + + + + + + + +
+ + + + + + + + + + + + +
Title1 year1 KB1020
+ + + """ + response = mock.Mock(text=html) + results = seedpeer.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 1) + self.assertEqual(results[0]['title'], 'Title') + self.assertEqual(results[0]['url'], 'https://seedpeer.me/link') + self.assertEqual(results[0]['seed'], 10) + self.assertEqual(results[0]['leech'], 20) + self.assertEqual(results[0]['filesize'], 1024) + self.assertEqual(results[0]['torrentfile'], 'https://seedpeer.me/torrent/abc123') + self.assertEqual(results[0]['magnetlink'], 'magnet:?xt=urn:btih:abc123')