OwlBurst/Ponysearch
1
0
Fork 0
forked from Ponysearch/Ponysearch
Code Pull requests Activity

Merge pull request #633 from kvch/deviantart-fix

Browse source 
[fix] deviantart engine xpaths
This commit is contained in:
Adam Tauber 2016-07-19 10:02:47 +02:00 committed by GitHub
commit 34a85533e0
2 changed files with 12 additions and 44 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
searx/engines/deviantart.py
View file

@ -50,11 +50,10 @@ def response(resp):
regex = re.compile(r'\/200H\/')
# parse results
for result in dom.xpath('//div[contains(@class, "tt-a tt-fh")]'):
link = result.xpath('.//a[contains(@class, "thumb")]')[0]
url = urljoin(base_url, link.attrib.get('href'))
title_links = result.xpath('.//span[@class="details"]//a[contains(@class, "t")]')
title = extract_text(title_links[0])
for result in dom.xpath('.//span[@class="thumb wide"]'):
link = result.xpath('.//a[@class="torpedo-thumb-link"]')[0]
url = link.attrib.get('href')
title = extract_text(result.xpath('.//span[@class="title"]'))
thumbnail_src = link.xpath('.//img')[0].attrib.get('src')
img_src = regex.sub('/', thumbnail_src)

47
tests/unit/engines/test_deviantart.py
View file

@ -28,44 +28,13 @@ class TestDeviantartEngine(SearxTestCase):
self.assertEqual(deviantart.response(response), [])
html = """
<div class="tt-a tt-fh tt-boxed" collect_rid="1:149167425"
usericon="http://a.deviantart.net/avatars/t/e/test-0.gif" userid="233301"
username="test-0" symbol="~" category="digitalart/animation">
<span class="tt-w" style="width: auto; max-width: 277px;">
<span class="tt-fh-tc" style="width: 202px;">
<span class="tt-bb" style="width: 202px;">
</span>
<span class="shadow">
<a class="thumb" href="http://url.of.result/2nd.part.of.url"
title="Behoimi BE Animation Test by test-0, Jan 4,
2010 in Digital Art &gt; Animation"> <i></i>
<img width="200" height="200" alt="Test"
src="http://url.of.thumbnail" data-src="http://th08.deviantart.net/test.jpg">
</a>
</span>
<!-- ^TTT -->
</span>
<span class="details">
<a href="http://test-0.deviantart.com/art/Test" class="t"
title="Behoimi BE Animation Test by test-0, Jan 4, 2010">
<span class="tt-fh-oe">Title of image</span> </a>
<small>
<span class="category">
<span class="age">
5 years ago
</span>
in <a title="Behoimi BE Animation Test by test-0, Jan 4, 2010"
href="http://www.deviantart.com/browse/all/digitalart/animation/">Animation</a>
</span>
<div class="commentcount">
<a href="http://test-0.deviantart.com/art/Test#comments">
<span class="iconcommentsstats"></span>9 Comments</a>
</div>
<a class="mlt-link" href="http://www.deviantart.com/morelikethis/149167425">
<span class="mlt-icon"></span> <span class="mlt-text">More Like This</span> </a>
</span>
</small> <!-- TTT$ -->
</span>
<div id="page-1-results" class="page-results results-page-thumb torpedo-container">
<span class="thumb wide" href="http://amai911.deviantart.com/art/Horse-195212845"
data-super-full-width="900" data-super-full-height="600">
<a class="torpedo-thumb-link" href="https://url.of.image">
<img data-sigil="torpedo-img" src="https://url.of.thumbnail" />
</a>
<span class="info"><span class="title-wrap"><span class="title">Title of image</span></span>
</div>
"""
response = mock.Mock(text=html)
@ -73,7 +42,7 @@ class TestDeviantartEngine(SearxTestCase):
self.assertEqual(type(results), list)
self.assertEqual(len(results), 1)
self.assertEqual(results[0]['title'], 'Title of image')
self.assertEqual(results[0]['url'], 'http://url.of.result/2nd.part.of.url')
self.assertEqual(results[0]['url'], 'https://url.of.image')
self.assertNotIn('content', results[0])
self.assertEqual(results[0]['thumbnail_src'], 'https://url.of.thumbnail')