From 448bfe6005baabb647bf02c9f24f9c7986b74c2e Mon Sep 17 00:00:00 2001
From: Marc Abonce Seguin <marc-abonce@mailbox.org>
Date: Sun, 2 May 2021 17:24:28 -0700
Subject: [PATCH] fix Qwant's fetch_languages function

---
 searx/data/engines_languages.json | 611 +++++++-----------------------
 searx/engines/qwant.py            |  16 +-
 2 files changed, 145 insertions(+), 482 deletions(-)

diff --git a/searx/data/engines_languages.json b/searx/data/engines_languages.json
index 08504935c..d4fc97f8b 100644
--- a/searx/data/engines_languages.json
+++ b/searx/data/engines_languages.json
@@ -25650,480 +25650,135 @@
     "zh",
     "zu"
   ],
-  "qwant": {
-    "bg-BG": {
-      "name": "\u0431\u044a\u043b\u0433\u0430\u0440\u0441\u043a\u0438 \u0435\u0437\u0438\u043a"
-    },
-    "br-FR": {
-      "name": "Brezhoneg"
-    },
-    "ca-AD": {
-      "name": "Catal\u00e0"
-    },
-    "ca-ES": {
-      "name": "Catal\u00e0"
-    },
-    "ca-FR": {
-      "name": "Catal\u00e0"
-    },
-    "co-FR": {
-      "name": "Corsu"
-    },
-    "cs-CZ": {
-      "name": "\u010cesky"
-    },
-    "cy-GB": {
-      "name": "Welsh"
-    },
-    "da-DK": {
-      "name": "Dansk"
-    },
-    "de-AT": {
-      "name": "Deutsch"
-    },
-    "de-CH": {
-      "name": "Deutsch"
-    },
-    "de-DE": {
-      "name": "Deutsch"
-    },
-    "el-GR": {
-      "name": "\u0395\u03bb\u03bb\u03b7\u03bd\u03b9\u03ba\u03ac"
-    },
-    "en-AU": {
-      "name": "English"
-    },
-    "en-CA": {
-      "name": "English"
-    },
-    "en-GB": {
-      "name": "English"
-    },
-    "en-IE": {
-      "name": "English"
-    },
-    "en-IN": {
-      "name": "English"
-    },
-    "en-MY": {
-      "name": "English"
-    },
-    "en-NZ": {
-      "name": "English"
-    },
-    "en-US": {
-      "name": "English"
-    },
-    "es-AD": {
-      "name": "Espa\u00f1ol"
-    },
-    "es-AR": {
-      "name": "Espa\u00f1ol"
-    },
-    "es-CL": {
-      "name": "Espa\u00f1ol"
-    },
-    "es-ES": {
-      "name": "Espa\u00f1ol"
-    },
-    "es-MX": {
-      "name": "Espa\u00f1ol"
-    },
-    "et-EE": {
-      "name": "Eesti keel"
-    },
-    "eu-ES": {
-      "name": "Euskara"
-    },
-    "eu-FR": {
-      "name": "Euskara"
-    },
-    "fi-FI": {
-      "name": "Suomen kieli"
-    },
-    "fr-AD": {
-      "name": "Fran\u00e7ais"
-    },
-    "fr-BE": {
-      "name": "Fran\u00e7ais"
-    },
-    "fr-CA": {
-      "name": "Fran\u00e7ais"
-    },
-    "fr-CH": {
-      "name": "Fran\u00e7ais"
-    },
-    "fr-FR": {
-      "name": "Fran\u00e7ais"
-    },
-    "gd-GB": {
-      "name": "Scottish"
-    },
-    "hu-HU": {
-      "name": "magyar"
-    },
-    "it-CH": {
-      "name": "Italiano"
-    },
-    "it-IT": {
-      "name": "Italiano"
-    },
-    "ko-KR": {
-      "name": "\ud55c\uad6d\uc5b4"
-    },
-    "nb-NO": {
-      "name": "Norsk"
-    },
-    "nl-BE": {
-      "name": "Nederlands"
-    },
-    "nl-NL": {
-      "name": "Nederlands"
-    },
-    "pl-PL": {
-      "name": "Polski"
-    },
-    "pt-AD": {
-      "name": "Portugu\u00eas"
-    },
-    "pt-BR": {
-      "name": "Portugu\u00eas"
-    },
-    "pt-PT": {
-      "name": "Portugu\u00eas"
-    },
-    "ro-RO": {
-      "name": "Rom\u00e2n\u0103"
-    },
-    "sv-SE": {
-      "name": "Svenska"
-    },
-    "th-TH": {
-      "name": "\u0e44\u0e17\u0e22"
-    },
-    "zh-CN": {
-      "name": "\u4e2d\u6587"
-    },
-    "zh-HK": {
-      "name": "\u4e2d\u6587"
-    }
-  },
-  "qwant images": {
-    "bg-BG": {
-      "name": "\u0431\u044a\u043b\u0433\u0430\u0440\u0441\u043a\u0438 \u0435\u0437\u0438\u043a"
-    },
-    "br-FR": {
-      "name": "Brezhoneg"
-    },
-    "ca-AD": {
-      "name": "Catal\u00e0"
-    },
-    "ca-ES": {
-      "name": "Catal\u00e0"
-    },
-    "ca-FR": {
-      "name": "Catal\u00e0"
-    },
-    "co-FR": {
-      "name": "Corsu"
-    },
-    "cs-CZ": {
-      "name": "\u010cesky"
-    },
-    "cy-GB": {
-      "name": "Welsh"
-    },
-    "da-DK": {
-      "name": "Dansk"
-    },
-    "de-AT": {
-      "name": "Deutsch"
-    },
-    "de-CH": {
-      "name": "Deutsch"
-    },
-    "de-DE": {
-      "name": "Deutsch"
-    },
-    "el-GR": {
-      "name": "\u0395\u03bb\u03bb\u03b7\u03bd\u03b9\u03ba\u03ac"
-    },
-    "en-AU": {
-      "name": "English"
-    },
-    "en-CA": {
-      "name": "English"
-    },
-    "en-GB": {
-      "name": "English"
-    },
-    "en-IE": {
-      "name": "English"
-    },
-    "en-IN": {
-      "name": "English"
-    },
-    "en-MY": {
-      "name": "English"
-    },
-    "en-NZ": {
-      "name": "English"
-    },
-    "en-US": {
-      "name": "English"
-    },
-    "es-AD": {
-      "name": "Espa\u00f1ol"
-    },
-    "es-AR": {
-      "name": "Espa\u00f1ol"
-    },
-    "es-CL": {
-      "name": "Espa\u00f1ol"
-    },
-    "es-ES": {
-      "name": "Espa\u00f1ol"
-    },
-    "es-MX": {
-      "name": "Espa\u00f1ol"
-    },
-    "et-EE": {
-      "name": "Eesti keel"
-    },
-    "eu-ES": {
-      "name": "Euskara"
-    },
-    "eu-FR": {
-      "name": "Euskara"
-    },
-    "fi-FI": {
-      "name": "Suomen kieli"
-    },
-    "fr-AD": {
-      "name": "Fran\u00e7ais"
-    },
-    "fr-BE": {
-      "name": "Fran\u00e7ais"
-    },
-    "fr-CA": {
-      "name": "Fran\u00e7ais"
-    },
-    "fr-CH": {
-      "name": "Fran\u00e7ais"
-    },
-    "fr-FR": {
-      "name": "Fran\u00e7ais"
-    },
-    "gd-GB": {
-      "name": "Scottish"
-    },
-    "hu-HU": {
-      "name": "magyar"
-    },
-    "it-CH": {
-      "name": "Italiano"
-    },
-    "it-IT": {
-      "name": "Italiano"
-    },
-    "ko-KR": {
-      "name": "\ud55c\uad6d\uc5b4"
-    },
-    "nb-NO": {
-      "name": "Norsk"
-    },
-    "nl-BE": {
-      "name": "Nederlands"
-    },
-    "nl-NL": {
-      "name": "Nederlands"
-    },
-    "pl-PL": {
-      "name": "Polski"
-    },
-    "pt-AD": {
-      "name": "Portugu\u00eas"
-    },
-    "pt-BR": {
-      "name": "Portugu\u00eas"
-    },
-    "pt-PT": {
-      "name": "Portugu\u00eas"
-    },
-    "ro-RO": {
-      "name": "Rom\u00e2n\u0103"
-    },
-    "sv-SE": {
-      "name": "Svenska"
-    },
-    "th-TH": {
-      "name": "\u0e44\u0e17\u0e22"
-    },
-    "zh-CN": {
-      "name": "\u4e2d\u6587"
-    },
-    "zh-HK": {
-      "name": "\u4e2d\u6587"
-    }
-  },
-  "qwant news": {
-    "bg-BG": {
-      "name": "\u0431\u044a\u043b\u0433\u0430\u0440\u0441\u043a\u0438 \u0435\u0437\u0438\u043a"
-    },
-    "br-FR": {
-      "name": "Brezhoneg"
-    },
-    "ca-AD": {
-      "name": "Catal\u00e0"
-    },
-    "ca-ES": {
-      "name": "Catal\u00e0"
-    },
-    "ca-FR": {
-      "name": "Catal\u00e0"
-    },
-    "co-FR": {
-      "name": "Corsu"
-    },
-    "cs-CZ": {
-      "name": "\u010cesky"
-    },
-    "cy-GB": {
-      "name": "Welsh"
-    },
-    "da-DK": {
-      "name": "Dansk"
-    },
-    "de-AT": {
-      "name": "Deutsch"
-    },
-    "de-CH": {
-      "name": "Deutsch"
-    },
-    "de-DE": {
-      "name": "Deutsch"
-    },
-    "el-GR": {
-      "name": "\u0395\u03bb\u03bb\u03b7\u03bd\u03b9\u03ba\u03ac"
-    },
-    "en-AU": {
-      "name": "English"
-    },
-    "en-CA": {
-      "name": "English"
-    },
-    "en-GB": {
-      "name": "English"
-    },
-    "en-IE": {
-      "name": "English"
-    },
-    "en-IN": {
-      "name": "English"
-    },
-    "en-MY": {
-      "name": "English"
-    },
-    "en-NZ": {
-      "name": "English"
-    },
-    "en-US": {
-      "name": "English"
-    },
-    "es-AD": {
-      "name": "Espa\u00f1ol"
-    },
-    "es-AR": {
-      "name": "Espa\u00f1ol"
-    },
-    "es-CL": {
-      "name": "Espa\u00f1ol"
-    },
-    "es-ES": {
-      "name": "Espa\u00f1ol"
-    },
-    "es-MX": {
-      "name": "Espa\u00f1ol"
-    },
-    "et-EE": {
-      "name": "Eesti keel"
-    },
-    "eu-ES": {
-      "name": "Euskara"
-    },
-    "eu-FR": {
-      "name": "Euskara"
-    },
-    "fi-FI": {
-      "name": "Suomen kieli"
-    },
-    "fr-AD": {
-      "name": "Fran\u00e7ais"
-    },
-    "fr-BE": {
-      "name": "Fran\u00e7ais"
-    },
-    "fr-CA": {
-      "name": "Fran\u00e7ais"
-    },
-    "fr-CH": {
-      "name": "Fran\u00e7ais"
-    },
-    "fr-FR": {
-      "name": "Fran\u00e7ais"
-    },
-    "gd-GB": {
-      "name": "Scottish"
-    },
-    "hu-HU": {
-      "name": "magyar"
-    },
-    "it-CH": {
-      "name": "Italiano"
-    },
-    "it-IT": {
-      "name": "Italiano"
-    },
-    "ko-KR": {
-      "name": "\ud55c\uad6d\uc5b4"
-    },
-    "nb-NO": {
-      "name": "Norsk"
-    },
-    "nl-BE": {
-      "name": "Nederlands"
-    },
-    "nl-NL": {
-      "name": "Nederlands"
-    },
-    "pl-PL": {
-      "name": "Polski"
-    },
-    "pt-AD": {
-      "name": "Portugu\u00eas"
-    },
-    "pt-BR": {
-      "name": "Portugu\u00eas"
-    },
-    "pt-PT": {
-      "name": "Portugu\u00eas"
-    },
-    "ro-RO": {
-      "name": "Rom\u00e2n\u0103"
-    },
-    "sv-SE": {
-      "name": "Svenska"
-    },
-    "th-TH": {
-      "name": "\u0e44\u0e17\u0e22"
-    },
-    "zh-CN": {
-      "name": "\u4e2d\u6587"
-    },
-    "zh-HK": {
-      "name": "\u4e2d\u6587"
-    }
-  },
+  "qwant": [
+    "bg-BG",
+    "ca-ES",
+    "cs-CZ",
+    "da-DK",
+    "de-AT",
+    "de-CH",
+    "de-DE",
+    "el-GR",
+    "en-AU",
+    "en-CA",
+    "en-GB",
+    "en-IE",
+    "en-IN",
+    "en-MY",
+    "en-NZ",
+    "en-US",
+    "es-AR",
+    "es-CL",
+    "es-ES",
+    "es-MX",
+    "et-EE",
+    "fi-FI",
+    "fr-BE",
+    "fr-CA",
+    "fr-CH",
+    "fr-FR",
+    "hu-HU",
+    "it-CH",
+    "it-IT",
+    "ko-KR",
+    "nb-NO",
+    "nl-BE",
+    "nl-NL",
+    "pl-PL",
+    "pt-BR",
+    "pt-PT",
+    "ro-RO",
+    "sv-SE",
+    "th-TH",
+    "zh-CN",
+    "zh-HK"
+  ],
+  "qwant images": [
+    "bg-BG",
+    "ca-ES",
+    "cs-CZ",
+    "da-DK",
+    "de-AT",
+    "de-CH",
+    "de-DE",
+    "el-GR",
+    "en-AU",
+    "en-CA",
+    "en-GB",
+    "en-IE",
+    "en-IN",
+    "en-MY",
+    "en-NZ",
+    "en-US",
+    "es-AR",
+    "es-CL",
+    "es-ES",
+    "es-MX",
+    "et-EE",
+    "fi-FI",
+    "fr-BE",
+    "fr-CA",
+    "fr-CH",
+    "fr-FR",
+    "hu-HU",
+    "it-CH",
+    "it-IT",
+    "ko-KR",
+    "nb-NO",
+    "nl-BE",
+    "nl-NL",
+    "pl-PL",
+    "pt-BR",
+    "pt-PT",
+    "ro-RO",
+    "sv-SE",
+    "th-TH",
+    "zh-CN",
+    "zh-HK"
+  ],
+  "qwant news": [
+    "bg-BG",
+    "ca-ES",
+    "cs-CZ",
+    "da-DK",
+    "de-AT",
+    "de-CH",
+    "de-DE",
+    "el-GR",
+    "en-AU",
+    "en-CA",
+    "en-GB",
+    "en-IE",
+    "en-IN",
+    "en-MY",
+    "en-NZ",
+    "en-US",
+    "es-AR",
+    "es-CL",
+    "es-ES",
+    "es-MX",
+    "et-EE",
+    "fi-FI",
+    "fr-BE",
+    "fr-CA",
+    "fr-CH",
+    "fr-FR",
+    "hu-HU",
+    "it-CH",
+    "it-IT",
+    "ko-KR",
+    "nb-NO",
+    "nl-BE",
+    "nl-NL",
+    "pl-PL",
+    "pt-BR",
+    "pt-PT",
+    "ro-RO",
+    "sv-SE",
+    "th-TH",
+    "zh-CN",
+    "zh-HK"
+  ],
   "startpage": {
     "af": {
       "alias": "afrikaans"
@@ -27428,6 +27083,10 @@
       "english_name": "Tamil",
       "name": "\u0ba4\u0bae\u0bbf\u0bb4\u0bcd"
     },
+    "tay": {
+      "english_name": "Tayal",
+      "name": "Tayal"
+    },
     "tcy": {
       "english_name": "Tulu",
       "name": "\u0ca4\u0cc1\u0cb3\u0cc1"
@@ -28666,6 +28325,10 @@
       "english_name": "Tamil",
       "name": "\u0ba4\u0bae\u0bbf\u0bb4\u0bcd"
     },
+    "tay": {
+      "english_name": "Tayal",
+      "name": "Tayal"
+    },
     "tcy": {
       "english_name": "Tulu",
       "name": "\u0ca4\u0cc1\u0cb3\u0cc1"
diff --git a/searx/engines/qwant.py b/searx/engines/qwant.py
index d01dc0acc..fb525843b 100644
--- a/searx/engines/qwant.py
+++ b/searx/engines/qwant.py
@@ -23,7 +23,7 @@ about = {
 # engine dependent config
 categories = []
 paging = True
-supported_languages_url = 'https://qwant.com/region'
+supported_languages_url = about['website']
 
 category_to_keyword = {'general': 'web',
                        'images': 'images',
@@ -125,15 +125,15 @@ def response(resp):
 def _fetch_supported_languages(resp):
     # list of regions is embedded in page as a js object
     response_text = resp.text
-    response_text = response_text[response_text.find('regionalisation'):]
-    response_text = response_text[response_text.find('{'):response_text.find(');')]
+    response_text = response_text[response_text.find('INITIAL_PROPS'):]
+    response_text = response_text[response_text.find('{'):response_text.find('</script>')]
 
     regions_json = loads(response_text)
 
-    supported_languages = {}
-    for lang in regions_json['languages'].values():
-        for country in lang['countries']:
-            lang_code = "{lang}-{country}".format(lang=lang['code'], country=country)
-            supported_languages[lang_code] = {'name': lang['name']}
+    supported_languages = []
+    for country, langs in regions_json['locales'].items():
+        for lang in langs['langs']:
+            lang_code = "{lang}-{country}".format(lang=lang, country=country)
+            supported_languages.append(lang_code)
 
     return supported_languages