From ec4237586db661f023df30f0ffee2a7f8761ceaa Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Mon, 26 Dec 2022 08:30:03 +0000 Subject: [PATCH 1/2] Use fasttext-predict instead of fasttext or fasttext-wheel See https://github.com/searxng/fasttext-predict --- Dockerfile | 3 --- requirements.txt | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index ece20c86b..66f58395d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -36,7 +36,6 @@ RUN apk add --no-cache -t build-dependencies \ su-exec \ python3 \ py3-pip \ - py3-numpy \ libxml2 \ libxslt \ openssl \ @@ -44,8 +43,6 @@ RUN apk add --no-cache -t build-dependencies \ uwsgi \ uwsgi-python3 \ brotli \ - && pip3 install --no-cache setuptools wheel \ - && sed -i s/fasttext-wheel/fasttext/ requirements.txt \ && pip3 install --no-cache -r requirements.txt \ && apk del build-dependencies \ && rm -rf /root/.cache diff --git a/requirements.txt b/requirements.txt index 5c45fff6b..ea2616e1a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,4 +15,4 @@ setproctitle==1.3.2 redis==4.4.0 markdown-it-py==2.1.0 typing_extensions==4.4.0 -fasttext-wheel==0.9.2 +fasttext-predict==0.9.2.1 From f3515041af527741cbdc383434d099c38e40dfba Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Mon, 26 Dec 2022 08:32:58 +0000 Subject: [PATCH 2/2] Lazy load fasttext-predict --- searx/utils.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/searx/utils.py b/searx/utils.py index 2157a4ce0..cda336035 100644 --- a/searx/utils.py +++ b/searx/utils.py @@ -15,7 +15,6 @@ from os.path import splitext, join from random import choice from html.parser import HTMLParser from urllib.parse import urljoin, urlparse -import fasttext from lxml import html from lxml.etree import ElementBase, XPath, XPathError, XPathSyntaxError, _ElementStringResult, _ElementUnicodeResult @@ -51,12 +50,9 @@ _STORAGE_UNIT_VALUE: Dict[str, int] = { _XPATH_CACHE: Dict[str, XPath] = {} _LANG_TO_LC_CACHE: Dict[str, Dict[str, str]] = {} -_FASTTEXT_MODEL: Optional[fasttext.FastText._FastText] = None +_FASTTEXT_MODEL: Optional["fasttext.FastText._FastText"] = None """fasttext model to predict laguage of a search term""" -# Monkey patch: prevent fasttext from showing a (useless) warning when loading a model. -fasttext.FastText.eprint = lambda x: None - class _NotSetClass: # pylint: disable=too-few-public-methods """Internal class for this module, do not create instance of this class. @@ -630,9 +626,13 @@ def eval_xpath_getindex(elements: ElementBase, xpath_spec: XPathSpecType, index: return default -def _get_fasttext_model() -> fasttext.FastText._FastText: +def _get_fasttext_model() -> "fasttext.FastText._FastText": global _FASTTEXT_MODEL # pylint: disable=global-statement if _FASTTEXT_MODEL is None: + import fasttext # pylint: disable=import-outside-toplevel + + # Monkey patch: prevent fasttext from showing a (useless) warning when loading a model. + fasttext.FastText.eprint = lambda x: None _FASTTEXT_MODEL = fasttext.load_model(str(data_dir / 'lid.176.ftz')) return _FASTTEXT_MODEL