fix Qwant's fetch_languages function

This commit is contained in:
Marc Abonce Seguin 2021-05-02 17:24:28 -07:00
parent 2756e170ce
commit 448bfe6005
2 changed files with 145 additions and 482 deletions

View file

@ -25650,480 +25650,135 @@
"zh", "zh",
"zu" "zu"
], ],
"qwant": { "qwant": [
"bg-BG": { "bg-BG",
"name": "\u0431\u044a\u043b\u0433\u0430\u0440\u0441\u043a\u0438 \u0435\u0437\u0438\u043a" "ca-ES",
}, "cs-CZ",
"br-FR": { "da-DK",
"name": "Brezhoneg" "de-AT",
}, "de-CH",
"ca-AD": { "de-DE",
"name": "Catal\u00e0" "el-GR",
}, "en-AU",
"ca-ES": { "en-CA",
"name": "Catal\u00e0" "en-GB",
}, "en-IE",
"ca-FR": { "en-IN",
"name": "Catal\u00e0" "en-MY",
}, "en-NZ",
"co-FR": { "en-US",
"name": "Corsu" "es-AR",
}, "es-CL",
"cs-CZ": { "es-ES",
"name": "\u010cesky" "es-MX",
}, "et-EE",
"cy-GB": { "fi-FI",
"name": "Welsh" "fr-BE",
}, "fr-CA",
"da-DK": { "fr-CH",
"name": "Dansk" "fr-FR",
}, "hu-HU",
"de-AT": { "it-CH",
"name": "Deutsch" "it-IT",
}, "ko-KR",
"de-CH": { "nb-NO",
"name": "Deutsch" "nl-BE",
}, "nl-NL",
"de-DE": { "pl-PL",
"name": "Deutsch" "pt-BR",
}, "pt-PT",
"el-GR": { "ro-RO",
"name": "\u0395\u03bb\u03bb\u03b7\u03bd\u03b9\u03ba\u03ac" "sv-SE",
}, "th-TH",
"en-AU": { "zh-CN",
"name": "English" "zh-HK"
}, ],
"en-CA": { "qwant images": [
"name": "English" "bg-BG",
}, "ca-ES",
"en-GB": { "cs-CZ",
"name": "English" "da-DK",
}, "de-AT",
"en-IE": { "de-CH",
"name": "English" "de-DE",
}, "el-GR",
"en-IN": { "en-AU",
"name": "English" "en-CA",
}, "en-GB",
"en-MY": { "en-IE",
"name": "English" "en-IN",
}, "en-MY",
"en-NZ": { "en-NZ",
"name": "English" "en-US",
}, "es-AR",
"en-US": { "es-CL",
"name": "English" "es-ES",
}, "es-MX",
"es-AD": { "et-EE",
"name": "Espa\u00f1ol" "fi-FI",
}, "fr-BE",
"es-AR": { "fr-CA",
"name": "Espa\u00f1ol" "fr-CH",
}, "fr-FR",
"es-CL": { "hu-HU",
"name": "Espa\u00f1ol" "it-CH",
}, "it-IT",
"es-ES": { "ko-KR",
"name": "Espa\u00f1ol" "nb-NO",
}, "nl-BE",
"es-MX": { "nl-NL",
"name": "Espa\u00f1ol" "pl-PL",
}, "pt-BR",
"et-EE": { "pt-PT",
"name": "Eesti keel" "ro-RO",
}, "sv-SE",
"eu-ES": { "th-TH",
"name": "Euskara" "zh-CN",
}, "zh-HK"
"eu-FR": { ],
"name": "Euskara" "qwant news": [
}, "bg-BG",
"fi-FI": { "ca-ES",
"name": "Suomen kieli" "cs-CZ",
}, "da-DK",
"fr-AD": { "de-AT",
"name": "Fran\u00e7ais" "de-CH",
}, "de-DE",
"fr-BE": { "el-GR",
"name": "Fran\u00e7ais" "en-AU",
}, "en-CA",
"fr-CA": { "en-GB",
"name": "Fran\u00e7ais" "en-IE",
}, "en-IN",
"fr-CH": { "en-MY",
"name": "Fran\u00e7ais" "en-NZ",
}, "en-US",
"fr-FR": { "es-AR",
"name": "Fran\u00e7ais" "es-CL",
}, "es-ES",
"gd-GB": { "es-MX",
"name": "Scottish" "et-EE",
}, "fi-FI",
"hu-HU": { "fr-BE",
"name": "magyar" "fr-CA",
}, "fr-CH",
"it-CH": { "fr-FR",
"name": "Italiano" "hu-HU",
}, "it-CH",
"it-IT": { "it-IT",
"name": "Italiano" "ko-KR",
}, "nb-NO",
"ko-KR": { "nl-BE",
"name": "\ud55c\uad6d\uc5b4" "nl-NL",
}, "pl-PL",
"nb-NO": { "pt-BR",
"name": "Norsk" "pt-PT",
}, "ro-RO",
"nl-BE": { "sv-SE",
"name": "Nederlands" "th-TH",
}, "zh-CN",
"nl-NL": { "zh-HK"
"name": "Nederlands" ],
},
"pl-PL": {
"name": "Polski"
},
"pt-AD": {
"name": "Portugu\u00eas"
},
"pt-BR": {
"name": "Portugu\u00eas"
},
"pt-PT": {
"name": "Portugu\u00eas"
},
"ro-RO": {
"name": "Rom\u00e2n\u0103"
},
"sv-SE": {
"name": "Svenska"
},
"th-TH": {
"name": "\u0e44\u0e17\u0e22"
},
"zh-CN": {
"name": "\u4e2d\u6587"
},
"zh-HK": {
"name": "\u4e2d\u6587"
}
},
"qwant images": {
"bg-BG": {
"name": "\u0431\u044a\u043b\u0433\u0430\u0440\u0441\u043a\u0438 \u0435\u0437\u0438\u043a"
},
"br-FR": {
"name": "Brezhoneg"
},
"ca-AD": {
"name": "Catal\u00e0"
},
"ca-ES": {
"name": "Catal\u00e0"
},
"ca-FR": {
"name": "Catal\u00e0"
},
"co-FR": {
"name": "Corsu"
},
"cs-CZ": {
"name": "\u010cesky"
},
"cy-GB": {
"name": "Welsh"
},
"da-DK": {
"name": "Dansk"
},
"de-AT": {
"name": "Deutsch"
},
"de-CH": {
"name": "Deutsch"
},
"de-DE": {
"name": "Deutsch"
},
"el-GR": {
"name": "\u0395\u03bb\u03bb\u03b7\u03bd\u03b9\u03ba\u03ac"
},
"en-AU": {
"name": "English"
},
"en-CA": {
"name": "English"
},
"en-GB": {
"name": "English"
},
"en-IE": {
"name": "English"
},
"en-IN": {
"name": "English"
},
"en-MY": {
"name": "English"
},
"en-NZ": {
"name": "English"
},
"en-US": {
"name": "English"
},
"es-AD": {
"name": "Espa\u00f1ol"
},
"es-AR": {
"name": "Espa\u00f1ol"
},
"es-CL": {
"name": "Espa\u00f1ol"
},
"es-ES": {
"name": "Espa\u00f1ol"
},
"es-MX": {
"name": "Espa\u00f1ol"
},
"et-EE": {
"name": "Eesti keel"
},
"eu-ES": {
"name": "Euskara"
},
"eu-FR": {
"name": "Euskara"
},
"fi-FI": {
"name": "Suomen kieli"
},
"fr-AD": {
"name": "Fran\u00e7ais"
},
"fr-BE": {
"name": "Fran\u00e7ais"
},
"fr-CA": {
"name": "Fran\u00e7ais"
},
"fr-CH": {
"name": "Fran\u00e7ais"
},
"fr-FR": {
"name": "Fran\u00e7ais"
},
"gd-GB": {
"name": "Scottish"
},
"hu-HU": {
"name": "magyar"
},
"it-CH": {
"name": "Italiano"
},
"it-IT": {
"name": "Italiano"
},
"ko-KR": {
"name": "\ud55c\uad6d\uc5b4"
},
"nb-NO": {
"name": "Norsk"
},
"nl-BE": {
"name": "Nederlands"
},
"nl-NL": {
"name": "Nederlands"
},
"pl-PL": {
"name": "Polski"
},
"pt-AD": {
"name": "Portugu\u00eas"
},
"pt-BR": {
"name": "Portugu\u00eas"
},
"pt-PT": {
"name": "Portugu\u00eas"
},
"ro-RO": {
"name": "Rom\u00e2n\u0103"
},
"sv-SE": {
"name": "Svenska"
},
"th-TH": {
"name": "\u0e44\u0e17\u0e22"
},
"zh-CN": {
"name": "\u4e2d\u6587"
},
"zh-HK": {
"name": "\u4e2d\u6587"
}
},
"qwant news": {
"bg-BG": {
"name": "\u0431\u044a\u043b\u0433\u0430\u0440\u0441\u043a\u0438 \u0435\u0437\u0438\u043a"
},
"br-FR": {
"name": "Brezhoneg"
},
"ca-AD": {
"name": "Catal\u00e0"
},
"ca-ES": {
"name": "Catal\u00e0"
},
"ca-FR": {
"name": "Catal\u00e0"
},
"co-FR": {
"name": "Corsu"
},
"cs-CZ": {
"name": "\u010cesky"
},
"cy-GB": {
"name": "Welsh"
},
"da-DK": {
"name": "Dansk"
},
"de-AT": {
"name": "Deutsch"
},
"de-CH": {
"name": "Deutsch"
},
"de-DE": {
"name": "Deutsch"
},
"el-GR": {
"name": "\u0395\u03bb\u03bb\u03b7\u03bd\u03b9\u03ba\u03ac"
},
"en-AU": {
"name": "English"
},
"en-CA": {
"name": "English"
},
"en-GB": {
"name": "English"
},
"en-IE": {
"name": "English"
},
"en-IN": {
"name": "English"
},
"en-MY": {
"name": "English"
},
"en-NZ": {
"name": "English"
},
"en-US": {
"name": "English"
},
"es-AD": {
"name": "Espa\u00f1ol"
},
"es-AR": {
"name": "Espa\u00f1ol"
},
"es-CL": {
"name": "Espa\u00f1ol"
},
"es-ES": {
"name": "Espa\u00f1ol"
},
"es-MX": {
"name": "Espa\u00f1ol"
},
"et-EE": {
"name": "Eesti keel"
},
"eu-ES": {
"name": "Euskara"
},
"eu-FR": {
"name": "Euskara"
},
"fi-FI": {
"name": "Suomen kieli"
},
"fr-AD": {
"name": "Fran\u00e7ais"
},
"fr-BE": {
"name": "Fran\u00e7ais"
},
"fr-CA": {
"name": "Fran\u00e7ais"
},
"fr-CH": {
"name": "Fran\u00e7ais"
},
"fr-FR": {
"name": "Fran\u00e7ais"
},
"gd-GB": {
"name": "Scottish"
},
"hu-HU": {
"name": "magyar"
},
"it-CH": {
"name": "Italiano"
},
"it-IT": {
"name": "Italiano"
},
"ko-KR": {
"name": "\ud55c\uad6d\uc5b4"
},
"nb-NO": {
"name": "Norsk"
},
"nl-BE": {
"name": "Nederlands"
},
"nl-NL": {
"name": "Nederlands"
},
"pl-PL": {
"name": "Polski"
},
"pt-AD": {
"name": "Portugu\u00eas"
},
"pt-BR": {
"name": "Portugu\u00eas"
},
"pt-PT": {
"name": "Portugu\u00eas"
},
"ro-RO": {
"name": "Rom\u00e2n\u0103"
},
"sv-SE": {
"name": "Svenska"
},
"th-TH": {
"name": "\u0e44\u0e17\u0e22"
},
"zh-CN": {
"name": "\u4e2d\u6587"
},
"zh-HK": {
"name": "\u4e2d\u6587"
}
},
"startpage": { "startpage": {
"af": { "af": {
"alias": "afrikaans" "alias": "afrikaans"
@ -27428,6 +27083,10 @@
"english_name": "Tamil", "english_name": "Tamil",
"name": "\u0ba4\u0bae\u0bbf\u0bb4\u0bcd" "name": "\u0ba4\u0bae\u0bbf\u0bb4\u0bcd"
}, },
"tay": {
"english_name": "Tayal",
"name": "Tayal"
},
"tcy": { "tcy": {
"english_name": "Tulu", "english_name": "Tulu",
"name": "\u0ca4\u0cc1\u0cb3\u0cc1" "name": "\u0ca4\u0cc1\u0cb3\u0cc1"
@ -28666,6 +28325,10 @@
"english_name": "Tamil", "english_name": "Tamil",
"name": "\u0ba4\u0bae\u0bbf\u0bb4\u0bcd" "name": "\u0ba4\u0bae\u0bbf\u0bb4\u0bcd"
}, },
"tay": {
"english_name": "Tayal",
"name": "Tayal"
},
"tcy": { "tcy": {
"english_name": "Tulu", "english_name": "Tulu",
"name": "\u0ca4\u0cc1\u0cb3\u0cc1" "name": "\u0ca4\u0cc1\u0cb3\u0cc1"

View file

@ -23,7 +23,7 @@ about = {
# engine dependent config # engine dependent config
categories = [] categories = []
paging = True paging = True
supported_languages_url = 'https://qwant.com/region' supported_languages_url = about['website']
category_to_keyword = {'general': 'web', category_to_keyword = {'general': 'web',
'images': 'images', 'images': 'images',
@ -125,15 +125,15 @@ def response(resp):
def _fetch_supported_languages(resp): def _fetch_supported_languages(resp):
# list of regions is embedded in page as a js object # list of regions is embedded in page as a js object
response_text = resp.text response_text = resp.text
response_text = response_text[response_text.find('regionalisation'):] response_text = response_text[response_text.find('INITIAL_PROPS'):]
response_text = response_text[response_text.find('{'):response_text.find(');')] response_text = response_text[response_text.find('{'):response_text.find('</script>')]
regions_json = loads(response_text) regions_json = loads(response_text)
supported_languages = {} supported_languages = []
for lang in regions_json['languages'].values(): for country, langs in regions_json['locales'].items():
for country in lang['countries']: for lang in langs['langs']:
lang_code = "{lang}-{country}".format(lang=lang['code'], country=country) lang_code = "{lang}-{country}".format(lang=lang, country=country)
supported_languages[lang_code] = {'name': lang['name']} supported_languages.append(lang_code)
return supported_languages return supported_languages