OwlBurst/Ponysearch
1
0
Fork 0
forked from Ponysearch/Ponysearch
Code Pull requests Activity

Merge pull request #1068 from Apply55gx/genius

Browse source 
Continuation of PR #1004 (seems abandoned)
This commit is contained in:
Adam Tauber 2017-11-03 20:21:25 +01:00 committed by GitHub
commit 00a7041daa
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 323 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

88
searx/engines/genius.py Normal file
View file

@ -0,0 +1,88 @@
"""
Genius
@website https://www.genius.com/
@provide-api yes (https://docs.genius.com/)
@using-api yes
@results JSON
@stable yes
@parse url, title, content, thumbnail, publishedDate
"""
from json import loads
from searx.url_utils import urlencode
from datetime import datetime
# engine dependent config
categories = ['music']
paging = True
language_support = False
page_size = 5
url = 'https://genius.com/api/'
search_url = url + 'search/{index}?{query}&page={pageno}&per_page={page_size}'
def request(query, params):
params['url'] = search_url.format(query=urlencode({'q': query}),
index='multi',
page_size=page_size,
pageno=params['pageno'])
return params
def parse_lyric(hit):
try:
content = hit['highlights'][0]['value']
except:
content = None
timestamp = hit['result']['lyrics_updated_at']
result = {'url': hit['result']['url'],
'title': hit['result']['full_title'],
'content': content,
'thumbnail': hit['result']['song_art_image_thumbnail_url'],
'template': 'videos.html'}
if timestamp:
result.update({'publishedDate': datetime.fromtimestamp(timestamp)})
return result
def parse_artist(hit):
result = {'url': hit['result']['url'],
'title': hit['result']['name'],
'content': None,
'thumbnail': hit['result']['image_url'],
'template': 'videos.html'}
return result
def parse_album(hit):
result = {'url': hit['result']['url'],
'title': hit['result']['full_title'],
'thumbnail': hit['result']['cover_art_url'],
# 'thumbnail': hit['result']['cover_art_thumbnail_url'],
'template': 'videos.html'}
try:
year = hit['result']['release_date_components']['year']
except:
pass
else:
if year:
result.update({'content': 'Released: {}'.format(year)})
return result
parse = {'lyric': parse_lyric, 'song': parse_lyric, 'artist': parse_artist, 'album': parse_album}
def response(resp):
results = []
json = loads(resp.text)
hits = [hit for section in json['response']['sections'] for hit in section['hits']]
for hit in hits:
try:
func = parse[hit['type']]
except KeyError:
continue
results.append(func(hit))
return results

4
searx/settings.yml
View file

@ -241,6 +241,10 @@ engines:
shortcut : frk shortcut : frk
disabled : True disabled : True
- name : genius
engine : genius
shortcut : gen
- name : gigablast - name : gigablast
engine : gigablast engine : gigablast
shortcut : gb shortcut : gb

231
tests/unit/engines/test_genius.py Normal file
View file

@ -0,0 +1,231 @@
from collections import defaultdict
import mock
from datetime import datetime
from searx.engines import genius
from searx.testing import SearxTestCase
class TestGeniusEngine(SearxTestCase):
def test_request(self):
query = 'test_query'
dicto = defaultdict(dict)
dicto['pageno'] = 1
params = genius.request(query, dicto)
self.assertTrue('url' in params)
self.assertTrue(query in params['url'])
self.assertTrue('genius.com' in params['url'])
def test_response(self):
json_empty = """
{
"meta": {
"status": 200
},
"response": {
"sections": [
{
"type": "top_hit",
"hits": []
},
{
"type": "song",
"hits": []
},
{
"type": "lyric",
"hits": []
},
{
"type": "artist",
"hits": []
},
{
"type": "album",
"hits": []
},
{
"type": "tag",
"hits": []
},
{
"type": "video",
"hits": []
},
{
"type": "article",
"hits": []
},
{
"type": "user",
"hits": []
}
]
}
}
"""
resp = mock.Mock(text=json_empty)
self.assertEqual(genius.response(resp), [])
json = """
{
"meta": {
"status": 200
},
"response": {
"sections": [
{
"type": "lyric",
"hits": [
{
"highlights": [
{
"property": "lyrics",
"value": "Sample lyrics",
"snippet": true,
"ranges": []
}
],
"index": "lyric",
"type": "song",
"result": {
"_type": "song",
"annotation_count": 45,
"api_path": "/songs/52916",
"full_title": "J't'emmerde by MC Jean Gab'1",
"header_image_thumbnail_url": "https://images.genius.com/xxx.300x300x1.jpg",
"header_image_url": "https://images.genius.com/ef9f736a86df3c3b1772f3fb7fbdb21c.1000x1000x1.jpg",
"id": 52916,
"instrumental": false,
"lyrics_owner_id": 15586,
"lyrics_state": "complete",
"lyrics_updated_at": 1498744545,
"path": "/Mc-jean-gab1-jtemmerde-lyrics",
"pyongs_count": 4,
"song_art_image_thumbnail_url": "https://images.genius.com/xxx.300x300x1.jpg",
"stats": {
"hot": false,
"unreviewed_annotations": 0,
"pageviews": 62490
},
"title": "J't'emmerde",
"title_with_featured": "J't'emmerde",
"updated_by_human_at": 1498744546,
"url": "https://genius.com/Mc-jean-gab1-jtemmerde-lyrics",
"primary_artist": {
"_type": "artist",
"api_path": "/artists/12691",
"header_image_url": "https://images.genius.com/c7847662a58f8c2b0f02a6e217d60907.960x657x1.jpg",
"id": 12691,
"image_url": "https://s3.amazonaws.com/rapgenius/Mc-jean-gab1.jpg",
"index_character": "m",
"is_meme_verified": false,
"is_verified": false,
"name": "MC Jean Gab'1",
"slug": "Mc-jean-gab1",
"url": "https://genius.com/artists/Mc-jean-gab1"
}
}
}
]
},
{
"type": "artist",
"hits": [
{
"highlights": [],
"index": "artist",
"type": "artist",
"result": {
"_type": "artist",
"api_path": "/artists/191580",
"header_image_url": "https://assets.genius.com/images/default_avatar_300.png?1503090542",
"id": 191580,
"image_url": "https://assets.genius.com/images/default_avatar_300.png?1503090542",
"index_character": "a",
"is_meme_verified": false,
"is_verified": false,
"name": "ASDF Guy",
"slug": "Asdf-guy",
"url": "https://genius.com/artists/Asdf-guy"
}
}
]
},
{
"type": "album",
"hits": [
{
"highlights": [],
"index": "album",
"type": "album",
"result": {
"_type": "album",
"api_path": "/albums/132332",
"cover_art_thumbnail_url": "https://images.genius.com/xxx.300x300x1.jpg",
"cover_art_url": "https://images.genius.com/xxx.600x600x1.jpg",
"full_title": "ASD by A Skylit Drive",
"id": 132332,
"name": "ASD",
"name_with_artist": "ASD (artist: A Skylit Drive)",
"release_date_components": {
"year": 2015,
"month": null,
"day": null
},
"url": "https://genius.com/albums/A-skylit-drive/Asd",
"artist": {
"_type": "artist",
"api_path": "/artists/48712",
"header_image_url": "https://images.genius.com/814c1551293172c56306d0e310c6aa89.620x400x1.jpg",
"id": 48712,
"image_url": "https://images.genius.com/814c1551293172c56306d0e310c6aa89.620x400x1.jpg",
"index_character": "s",
"is_meme_verified": false,
"is_verified": false,
"name": "A Skylit Drive",
"slug": "A-skylit-drive",
"url": "https://genius.com/artists/A-skylit-drive"
}
}
}
]
}
]
}
}
"""
resp = mock.Mock(text=json)
results = genius.response(resp)
self.assertEqual(len(results), 3)
self.assertEqual(type(results), list)
# check lyric parsing
r = results[0]
self.assertEqual(r['url'], 'https://genius.com/Mc-jean-gab1-jtemmerde-lyrics')
self.assertEqual(r['title'], "J't'emmerde by MC Jean Gab'1")
self.assertEqual(r['content'], "Sample lyrics")
self.assertEqual(r['template'], 'videos.html')
self.assertEqual(r['thumbnail'], 'https://images.genius.com/xxx.300x300x1.jpg')
created = datetime.fromtimestamp(1498744545)
self.assertEqual(r['publishedDate'], created)
# check artist parsing
r = results[1]
self.assertEqual(r['url'], 'https://genius.com/artists/Asdf-guy')
self.assertEqual(r['title'], "ASDF Guy")
self.assertEqual(r['content'], None)
self.assertEqual(r['template'], 'videos.html')
self.assertEqual(r['thumbnail'], 'https://assets.genius.com/images/default_avatar_300.png?1503090542')
# check album parsing
r = results[2]
self.assertEqual(r['url'], 'https://genius.com/albums/A-skylit-drive/Asd')
self.assertEqual(r['title'], "ASD by A Skylit Drive")
self.assertEqual(r['content'], "Released: 2015")
self.assertEqual(r['template'], 'videos.html')
self.assertEqual(r['thumbnail'], 'https://images.genius.com/xxx.600x600x1.jpg')