forked from Ponysearch/Ponysearch
New engine: Genius (lyrics)
This commit is contained in:
woorst
parent
5baad02c06
commit
2434c29dc5
3 changed files with 324 additions and 0 deletions
89
searx/engines/genius.py
Normal file
89
searx/engines/genius.py
Normal file
|
|
@ -0,0 +1,89 @@
|
||||||
|
"""
|
||||||
|
Genius
|
||||||
|
|
||||||
|
@website https://www.genius.com/
|
||||||
|
@provide-api yes (https://docs.genius.com/)
|
||||||
|
|
||||||
|
@using-api yes
|
||||||
|
@results JSON
|
||||||
|
@stable yes
|
||||||
|
@parse url, title, content, thumbnail, publishedDate
|
||||||
|
"""
|
||||||
|
|
||||||
|
from json import loads
|
||||||
|
from searx.url_utils import urlencode
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
# engine dependent config
|
||||||
|
categories = ['music']
|
||||||
|
paging = True
|
||||||
|
language_support = False
|
||||||
|
page_size = 5
|
||||||
|
|
||||||
|
indicies = ['top_hit', 'song', 'lyric', 'artist', 'album', 'tag', 'video', 'article', 'user']
|
||||||
|
url = 'https://genius.com/api/'
|
||||||
|
search_url = url + 'search/{index}?{query}&page={pageno}&per_page={page_size}'
|
||||||
|
|
||||||
|
|
||||||
|
def request(query, params):
|
||||||
|
params['url'] = search_url.format(query=urlencode({'q': query}),
|
||||||
|
index='multi',
|
||||||
|
page_size=page_size,
|
||||||
|
pageno=params['pageno'])
|
||||||
|
return params
|
||||||
|
|
||||||
|
|
||||||
|
def parse_lyric(hit):
|
||||||
|
try:
|
||||||
|
content = hit['highlights'][0]['value']
|
||||||
|
except:
|
||||||
|
content = None
|
||||||
|
timestamp = hit['result']['lyrics_updated_at']
|
||||||
|
result = {'url': hit['result']['url'],
|
||||||
|
'title': hit['result']['full_title'],
|
||||||
|
'content': content,
|
||||||
|
'thumbnail': hit['result']['song_art_image_thumbnail_url'],
|
||||||
|
'template': 'videos.html'}
|
||||||
|
if timestamp:
|
||||||
|
result.update({'publishedDate': datetime.fromtimestamp(timestamp)})
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def parse_artist(hit):
|
||||||
|
result = {'url': hit['result']['url'],
|
||||||
|
'title': hit['result']['name'],
|
||||||
|
'content': None,
|
||||||
|
'thumbnail': hit['result']['image_url'],
|
||||||
|
'template': 'videos.html'}
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def parse_album(hit):
|
||||||
|
result = {'url': hit['result']['url'],
|
||||||
|
'title': hit['result']['full_title'],
|
||||||
|
'thumbnail': hit['result']['cover_art_url'],
|
||||||
|
# 'thumbnail': hit['result']['cover_art_thumbnail_url'],
|
||||||
|
'template': 'videos.html'}
|
||||||
|
try:
|
||||||
|
year = hit['result']['release_date_components']['year']
|
||||||
|
except:
|
||||||
|
pass
|
||||||
|
else:
|
||||||
|
if year:
|
||||||
|
result.update({'content': 'Released: {}'.format(year)})
|
||||||
|
return result
|
||||||
|
|
||||||
|
parse = {'lyric': parse_lyric, 'song': parse_lyric, 'artist': parse_artist, 'album': parse_album}
|
||||||
|
|
||||||
|
|
||||||
|
def response(resp):
|
||||||
|
results = []
|
||||||
|
json = loads(resp.text)
|
||||||
|
hits = [hit for section in json['response']['sections'] for hit in section['hits']]
|
||||||
|
for hit in hits:
|
||||||
|
try:
|
||||||
|
func = parse[hit['type']]
|
||||||
|
except KeyError:
|
||||||
|
continue
|
||||||
|
results.append(func(hit))
|
||||||
|
return results
|
||||||
|
|
@ -236,6 +236,10 @@ engines:
|
||||||
shortcut : frk
|
shortcut : frk
|
||||||
disabled : True
|
disabled : True
|
||||||
|
|
||||||
|
- name : genius
|
||||||
|
engine : genius
|
||||||
|
shortcut : gen
|
||||||
|
|
||||||
- name : gigablast
|
- name : gigablast
|
||||||
engine : gigablast
|
engine : gigablast
|
||||||
shortcut : gb
|
shortcut : gb
|
||||||
|
|
|
||||||
231
tests/unit/engines/test_genius.py
Normal file
231
tests/unit/engines/test_genius.py
Normal file
|
|
@ -0,0 +1,231 @@
|
||||||
|
from collections import defaultdict
|
||||||
|
import mock
|
||||||
|
from datetime import datetime
|
||||||
|
from searx.engines import genius
|
||||||
|
from searx.testing import SearxTestCase
|
||||||
|
|
||||||
|
|
||||||
|
class TestGeniusEngine(SearxTestCase):
|
||||||
|
|
||||||
|
def test_request(self):
|
||||||
|
query = 'test_query'
|
||||||
|
dicto = defaultdict(dict)
|
||||||
|
dicto['pageno'] = 1
|
||||||
|
params = genius.request(query, dicto)
|
||||||
|
self.assertTrue('url' in params)
|
||||||
|
self.assertTrue(query in params['url'])
|
||||||
|
self.assertTrue('genius.com' in params['url'])
|
||||||
|
|
||||||
|
def test_response(self):
|
||||||
|
|
||||||
|
json_empty = """
|
||||||
|
{
|
||||||
|
"meta": {
|
||||||
|
"status": 200
|
||||||
|
},
|
||||||
|
"response": {
|
||||||
|
"sections": [
|
||||||
|
{
|
||||||
|
"type": "top_hit",
|
||||||
|
"hits": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "song",
|
||||||
|
"hits": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "lyric",
|
||||||
|
"hits": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "artist",
|
||||||
|
"hits": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "album",
|
||||||
|
"hits": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "tag",
|
||||||
|
"hits": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "video",
|
||||||
|
"hits": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "article",
|
||||||
|
"hits": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "user",
|
||||||
|
"hits": []
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
"""
|
||||||
|
|
||||||
|
resp = mock.Mock(text=json_empty)
|
||||||
|
self.assertEqual(genius.response(resp), [])
|
||||||
|
|
||||||
|
json = """
|
||||||
|
{
|
||||||
|
"meta": {
|
||||||
|
"status": 200
|
||||||
|
},
|
||||||
|
"response": {
|
||||||
|
"sections": [
|
||||||
|
{
|
||||||
|
"type": "lyric",
|
||||||
|
"hits": [
|
||||||
|
{
|
||||||
|
"highlights": [
|
||||||
|
{
|
||||||
|
"property": "lyrics",
|
||||||
|
"value": "Sample lyrics",
|
||||||
|
"snippet": true,
|
||||||
|
"ranges": []
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"index": "lyric",
|
||||||
|
"type": "song",
|
||||||
|
"result": {
|
||||||
|
"_type": "song",
|
||||||
|
"annotation_count": 45,
|
||||||
|
"api_path": "/songs/52916",
|
||||||
|
"full_title": "J't'emmerde by MC Jean Gab'1",
|
||||||
|
"header_image_thumbnail_url": "https://images.genius.com/ef9f736a86df3c3b1772f3fb7fbdb21c.300x300x1.jpg",
|
||||||
|
"header_image_url": "https://images.genius.com/ef9f736a86df3c3b1772f3fb7fbdb21c.1000x1000x1.jpg",
|
||||||
|
"id": 52916,
|
||||||
|
"instrumental": false,
|
||||||
|
"lyrics_owner_id": 15586,
|
||||||
|
"lyrics_state": "complete",
|
||||||
|
"lyrics_updated_at": 1498744545,
|
||||||
|
"path": "/Mc-jean-gab1-jtemmerde-lyrics",
|
||||||
|
"pyongs_count": 4,
|
||||||
|
"song_art_image_thumbnail_url": "https://images.genius.com/ef9f736a86df3c3b1772f3fb7fbdb21c.300x300x1.jpg",
|
||||||
|
"stats": {
|
||||||
|
"hot": false,
|
||||||
|
"unreviewed_annotations": 0,
|
||||||
|
"pageviews": 62490
|
||||||
|
},
|
||||||
|
"title": "J't'emmerde",
|
||||||
|
"title_with_featured": "J't'emmerde",
|
||||||
|
"updated_by_human_at": 1498744546,
|
||||||
|
"url": "https://genius.com/Mc-jean-gab1-jtemmerde-lyrics",
|
||||||
|
"primary_artist": {
|
||||||
|
"_type": "artist",
|
||||||
|
"api_path": "/artists/12691",
|
||||||
|
"header_image_url": "https://images.genius.com/c7847662a58f8c2b0f02a6e217d60907.960x657x1.jpg",
|
||||||
|
"id": 12691,
|
||||||
|
"image_url": "https://s3.amazonaws.com/rapgenius/Mc-jean-gab1.jpg",
|
||||||
|
"index_character": "m",
|
||||||
|
"is_meme_verified": false,
|
||||||
|
"is_verified": false,
|
||||||
|
"name": "MC Jean Gab'1",
|
||||||
|
"slug": "Mc-jean-gab1",
|
||||||
|
"url": "https://genius.com/artists/Mc-jean-gab1"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "artist",
|
||||||
|
"hits": [
|
||||||
|
{
|
||||||
|
"highlights": [],
|
||||||
|
"index": "artist",
|
||||||
|
"type": "artist",
|
||||||
|
"result": {
|
||||||
|
"_type": "artist",
|
||||||
|
"api_path": "/artists/191580",
|
||||||
|
"header_image_url": "https://assets.genius.com/images/default_avatar_300.png?1503090542",
|
||||||
|
"id": 191580,
|
||||||
|
"image_url": "https://assets.genius.com/images/default_avatar_300.png?1503090542",
|
||||||
|
"index_character": "a",
|
||||||
|
"is_meme_verified": false,
|
||||||
|
"is_verified": false,
|
||||||
|
"name": "ASDF Guy",
|
||||||
|
"slug": "Asdf-guy",
|
||||||
|
"url": "https://genius.com/artists/Asdf-guy"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "album",
|
||||||
|
"hits": [
|
||||||
|
{
|
||||||
|
"highlights": [],
|
||||||
|
"index": "album",
|
||||||
|
"type": "album",
|
||||||
|
"result": {
|
||||||
|
"_type": "album",
|
||||||
|
"api_path": "/albums/132332",
|
||||||
|
"cover_art_thumbnail_url": "https://images.genius.com/147d70434ba190b9b1c26b06aee87d17.300x300x1.jpg",
|
||||||
|
"cover_art_url": "https://images.genius.com/147d70434ba190b9b1c26b06aee87d17.600x600x1.jpg",
|
||||||
|
"full_title": "ASD by A Skylit Drive",
|
||||||
|
"id": 132332,
|
||||||
|
"name": "ASD",
|
||||||
|
"name_with_artist": "ASD (artist: A Skylit Drive)",
|
||||||
|
"release_date_components": {
|
||||||
|
"year": 2015,
|
||||||
|
"month": null,
|
||||||
|
"day": null
|
||||||
|
},
|
||||||
|
"url": "https://genius.com/albums/A-skylit-drive/Asd",
|
||||||
|
"artist": {
|
||||||
|
"_type": "artist",
|
||||||
|
"api_path": "/artists/48712",
|
||||||
|
"header_image_url": "https://images.genius.com/814c1551293172c56306d0e310c6aa89.620x400x1.jpg",
|
||||||
|
"id": 48712,
|
||||||
|
"image_url": "https://images.genius.com/814c1551293172c56306d0e310c6aa89.620x400x1.jpg",
|
||||||
|
"index_character": "s",
|
||||||
|
"is_meme_verified": false,
|
||||||
|
"is_verified": false,
|
||||||
|
"name": "A Skylit Drive",
|
||||||
|
"slug": "A-skylit-drive",
|
||||||
|
"url": "https://genius.com/artists/A-skylit-drive"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
"""
|
||||||
|
|
||||||
|
resp = mock.Mock(text=json)
|
||||||
|
results = genius.response(resp)
|
||||||
|
|
||||||
|
self.assertEqual(len(results), 3)
|
||||||
|
self.assertEqual(type(results), list)
|
||||||
|
|
||||||
|
# check lyric parsing
|
||||||
|
r = results[0]
|
||||||
|
self.assertEqual(r['url'], 'https://genius.com/Mc-jean-gab1-jtemmerde-lyrics')
|
||||||
|
self.assertEqual(r['title'], "J't'emmerde by MC Jean Gab'1")
|
||||||
|
self.assertEqual(r['content'], "Sample lyrics")
|
||||||
|
self.assertEqual(r['template'], 'videos.html')
|
||||||
|
self.assertEqual(r['thumbnail'], 'https://images.genius.com/ef9f736a86df3c3b1772f3fb7fbdb21c.300x300x1.jpg')
|
||||||
|
created = datetime.fromtimestamp(1498744545)
|
||||||
|
self.assertEqual(r['publishedDate'], created)
|
||||||
|
|
||||||
|
# check artist parsing
|
||||||
|
r = results[1]
|
||||||
|
self.assertEqual(r['url'], 'https://genius.com/artists/Asdf-guy')
|
||||||
|
self.assertEqual(r['title'], "ASDF Guy")
|
||||||
|
self.assertEqual(r['content'], None)
|
||||||
|
self.assertEqual(r['template'], 'videos.html')
|
||||||
|
self.assertEqual(r['thumbnail'], 'https://assets.genius.com/images/default_avatar_300.png?1503090542')
|
||||||
|
|
||||||
|
# check album parsing
|
||||||
|
r = results[2]
|
||||||
|
self.assertEqual(r['url'], 'https://genius.com/albums/A-skylit-drive/Asd')
|
||||||
|
self.assertEqual(r['title'], "ASD by A Skylit Drive")
|
||||||
|
self.assertEqual(r['content'], "Released: 2015")
|
||||||
|
self.assertEqual(r['template'], 'videos.html')
|
||||||
|
self.assertEqual(r['thumbnail'], 'https://images.genius.com/147d70434ba190b9b1c26b06aee87d17.600x600x1.jpg')
|
||||||
Loading…
Reference in a new issue