From 912f877dec3a210347fc1af0a9d8de9d5298e7e3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 24 Mar 2023 07:57:14 +0000 Subject: [PATCH 01/14] Bump ionicons from 6.1.3 to 7.1.0 in /searx/static/themes/simple Bumps [ionicons](https://github.com/ionic-team/ionicons) from 6.1.3 to 7.1.0. - [Release notes](https://github.com/ionic-team/ionicons/releases) - [Changelog](https://github.com/ionic-team/ionicons/blob/main/CHANGELOG.md) - [Commits](https://github.com/ionic-team/ionicons/compare/v6.1.3...v7.1.0) --- updated-dependencies: - dependency-name: ionicons dependency-type: direct:development update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- searx/static/themes/simple/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/searx/static/themes/simple/package.json b/searx/static/themes/simple/package.json index 530a96c88..4c1cd169a 100644 --- a/searx/static/themes/simple/package.json +++ b/searx/static/themes/simple/package.json @@ -11,7 +11,7 @@ "grunt-eslint": "^24.0.0", "grunt-stylelint": "^0.16.0", "grunt-image": "^6.4.0", - "ionicons": "^6.0.2", + "ionicons": "^7.1.0", "less": "^4.1.3", "less-plugin-clean-css": "^1.5.1", "sharp": "^0.31.0", From 270ad188978fbc7c16e61ea0b997687ccfa0e3ee Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Thu, 30 Mar 2023 21:04:53 +0200 Subject: [PATCH 02/14] [fix] engine flickr: adapt to the new data model from flicker's response Closes: https://github.com/searxng/searxng/issues/1879 Signed-off-by: Markus Heiser --- searx/engines/flickr_noapi.py | 75 +++++++++++++++++++++-------------- 1 file changed, 46 insertions(+), 29 deletions(-) diff --git a/searx/engines/flickr_noapi.py b/searx/engines/flickr_noapi.py index 4ff59fc52..5299c604f 100644 --- a/searx/engines/flickr_noapi.py +++ b/searx/engines/flickr_noapi.py @@ -1,14 +1,22 @@ # SPDX-License-Identifier: AGPL-3.0-or-later -""" - Flickr (Images) +# lint: pylint +"""Flickr (Images) + """ -from json import loads +from typing import TYPE_CHECKING + +import json from time import time import re from urllib.parse import urlencode from searx.utils import ecma_unescape, html_to_text +if TYPE_CHECKING: + import logging + + logger: logging.Logger + # about about = { "website": 'https://www.flickr.com', @@ -19,23 +27,24 @@ about = { "results": 'HTML', } +# engine dependent config categories = ['images'] - -url = 'https://www.flickr.com/' -search_url = url + 'search?{query}&page={page}' -time_range_url = '&min_upload_date={start}&max_upload_date={end}' -photo_url = 'https://www.flickr.com/photos/{userid}/{photoid}' -modelexport_re = re.compile(r"^\s*modelExport:\s*({.*}),$", re.M) -image_sizes = ('o', 'k', 'h', 'b', 'c', 'z', 'n', 'm', 't', 'q', 's') - paging = True time_range_support = True +safesearch = False + time_range_dict = { 'day': 60 * 60 * 24, 'week': 60 * 60 * 24 * 7, 'month': 60 * 60 * 24 * 7 * 4, 'year': 60 * 60 * 24 * 7 * 52, } +image_sizes = ('o', 'k', 'h', 'b', 'c', 'z', 'm', 'n', 't', 'q', 's') + +search_url = 'https://www.flickr.com/search?{query}&page={page}' +time_range_url = '&min_upload_date={start}&max_upload_date={end}' +photo_url = 'https://www.flickr.com/photos/{userid}/{photoid}' +modelexport_re = re.compile(r"^\s*modelExport:\s*({.*}),$", re.M) def build_flickr_url(user_id, photo_id): @@ -55,51 +64,59 @@ def request(query, params): return params -def response(resp): +def response(resp): # pylint: disable=too-many-branches results = [] matches = modelexport_re.search(resp.text) - if matches is None: return results match = matches.group(1) - model_export = loads(match) + model_export = json.loads(match) if 'legend' not in model_export: return results - legend = model_export['legend'] # handle empty page if not legend or not legend[0]: return results - for index in legend: - photo = model_export['main'][index[0]][int(index[1])][index[2]][index[3]][int(index[4])] + for x, index in enumerate(legend): + if len(index) != 8: + logger.debug("skip legend enty %s : %s", x, index) + continue + + photo = model_export['main'][index[0]][int(index[1])][index[2]][index[3]][index[4]][index[5]][int(index[6])][ + index[7] + ] author = ecma_unescape(photo.get('realname', '')) - source = ecma_unescape(photo.get('username', '')) + ' @ Flickr' + source = ecma_unescape(photo.get('username', '')) + if source: + source += ' @ Flickr' title = ecma_unescape(photo.get('title', '')) content = html_to_text(ecma_unescape(photo.get('description', ''))) img_src = None + # From the biggest to the lowest format + size_data = None for image_size in image_sizes: - if image_size in photo['sizes']: - img_src = photo['sizes'][image_size]['url'] - img_format = ( - 'jpg ' + str(photo['sizes'][image_size]['width']) + 'x' + str(photo['sizes'][image_size]['height']) - ) + if image_size in photo['sizes']['data']: + size_data = photo['sizes']['data'][image_size]['data'] break - if not img_src: - logger.debug('cannot find valid image size: {0}'.format(repr(photo))) + if not size_data: + logger.debug('cannot find valid image size: {0}'.format(repr(photo['sizes']['data']))) continue + img_src = size_data['url'] + img_format = f"{size_data['width']} x {size_data['height']}" + # For a bigger thumbnail, keep only the url_z, not the url_n - if 'n' in photo['sizes']: - thumbnail_src = photo['sizes']['n']['url'] - elif 'z' in photo['sizes']: - thumbnail_src = photo['sizes']['z']['url'] + if 'n' in photo['sizes']['data']: + thumbnail_src = photo['sizes']['data']['n']['data']['url'] + elif 'z' in photo['sizes']['data']: + thumbnail_src = photo['sizes']['data']['z']['data']['url'] else: thumbnail_src = img_src From c1c24fc231a676edadb6cd216b77533478094845 Mon Sep 17 00:00:00 2001 From: searxng-bot Date: Fri, 31 Mar 2023 07:07:59 +0000 Subject: [PATCH 03/14] [translations] update from Weblate 17ad1118 - 2023-03-29 - return42 61446791 - 2023-03-29 - return42 --- searx/translations/af/LC_MESSAGES/messages.mo | Bin 9502 -> 9502 bytes searx/translations/ar/LC_MESSAGES/messages.mo | Bin 22011 -> 22011 bytes searx/translations/bg/LC_MESSAGES/messages.mo | Bin 22241 -> 22241 bytes searx/translations/bn/LC_MESSAGES/messages.mo | Bin 16735 -> 16735 bytes searx/translations/bo/LC_MESSAGES/messages.mo | Bin 11614 -> 11614 bytes searx/translations/ca/LC_MESSAGES/messages.mo | Bin 17480 -> 17480 bytes searx/translations/cs/LC_MESSAGES/messages.mo | Bin 17442 -> 17442 bytes searx/translations/cy/LC_MESSAGES/messages.mo | Bin 7859 -> 7859 bytes searx/translations/da/LC_MESSAGES/messages.mo | Bin 17167 -> 17167 bytes searx/translations/de/LC_MESSAGES/messages.mo | Bin 18742 -> 18742 bytes searx/translations/dv/LC_MESSAGES/messages.mo | Bin 1467 -> 1467 bytes .../el_GR/LC_MESSAGES/messages.mo | Bin 25630 -> 25630 bytes searx/translations/en/LC_MESSAGES/messages.mo | Bin 445 -> 445 bytes searx/translations/eo/LC_MESSAGES/messages.mo | Bin 12649 -> 12649 bytes searx/translations/es/LC_MESSAGES/messages.mo | Bin 18819 -> 18819 bytes searx/translations/et/LC_MESSAGES/messages.mo | Bin 14346 -> 14346 bytes searx/translations/eu/LC_MESSAGES/messages.mo | Bin 10388 -> 10388 bytes .../fa_IR/LC_MESSAGES/messages.mo | Bin 20625 -> 20625 bytes searx/translations/fi/LC_MESSAGES/messages.mo | Bin 16614 -> 16614 bytes .../translations/fil/LC_MESSAGES/messages.mo | Bin 14443 -> 14443 bytes searx/translations/fr/LC_MESSAGES/messages.mo | Bin 18992 -> 18992 bytes searx/translations/gl/LC_MESSAGES/messages.mo | Bin 18075 -> 18075 bytes searx/translations/he/LC_MESSAGES/messages.mo | Bin 18050 -> 18050 bytes searx/translations/hr/LC_MESSAGES/messages.mo | Bin 17138 -> 17138 bytes searx/translations/hu/LC_MESSAGES/messages.mo | Bin 17362 -> 17362 bytes searx/translations/ia/LC_MESSAGES/messages.mo | Bin 8229 -> 8229 bytes searx/translations/id/LC_MESSAGES/messages.mo | Bin 18007 -> 18007 bytes searx/translations/it/LC_MESSAGES/messages.mo | Bin 18345 -> 18345 bytes searx/translations/ja/LC_MESSAGES/messages.mo | Bin 19869 -> 19869 bytes searx/translations/ko/LC_MESSAGES/messages.mo | Bin 18500 -> 18500 bytes searx/translations/lt/LC_MESSAGES/messages.mo | Bin 15562 -> 15562 bytes searx/translations/lv/LC_MESSAGES/messages.mo | Bin 8196 -> 8196 bytes searx/translations/ml/LC_MESSAGES/messages.mo | Bin 1032 -> 1032 bytes searx/translations/ms/LC_MESSAGES/messages.mo | Bin 8706 -> 8706 bytes .../nb_NO/LC_MESSAGES/messages.mo | Bin 16006 -> 16340 bytes .../nb_NO/LC_MESSAGES/messages.po | 8 +++++--- searx/translations/nl/LC_MESSAGES/messages.mo | Bin 18143 -> 18143 bytes searx/translations/oc/LC_MESSAGES/messages.mo | Bin 8535 -> 8578 bytes searx/translations/pa/LC_MESSAGES/messages.mo | Bin 433 -> 433 bytes .../translations/pap/LC_MESSAGES/messages.mo | Bin 2081 -> 2081 bytes searx/translations/pl/LC_MESSAGES/messages.mo | Bin 18699 -> 18699 bytes searx/translations/pt/LC_MESSAGES/messages.mo | Bin 17347 -> 17347 bytes .../pt_BR/LC_MESSAGES/messages.mo | Bin 18447 -> 18447 bytes searx/translations/ro/LC_MESSAGES/messages.mo | Bin 17015 -> 17015 bytes searx/translations/ru/LC_MESSAGES/messages.mo | Bin 23533 -> 23533 bytes searx/translations/si/LC_MESSAGES/messages.mo | Bin 5937 -> 5937 bytes searx/translations/sk/LC_MESSAGES/messages.mo | Bin 17457 -> 17457 bytes searx/translations/sl/LC_MESSAGES/messages.mo | Bin 16978 -> 16978 bytes searx/translations/sr/LC_MESSAGES/messages.mo | Bin 21418 -> 21418 bytes searx/translations/sv/LC_MESSAGES/messages.mo | Bin 17919 -> 17919 bytes .../translations/szl/LC_MESSAGES/messages.mo | Bin 17255 -> 17255 bytes searx/translations/ta/LC_MESSAGES/messages.mo | Bin 24229 -> 24230 bytes searx/translations/ta/LC_MESSAGES/messages.po | 19 +++++++++--------- searx/translations/te/LC_MESSAGES/messages.mo | Bin 22992 -> 22992 bytes searx/translations/th/LC_MESSAGES/messages.mo | Bin 23724 -> 23724 bytes searx/translations/tr/LC_MESSAGES/messages.mo | Bin 18402 -> 18402 bytes searx/translations/uk/LC_MESSAGES/messages.mo | Bin 17794 -> 17794 bytes searx/translations/vi/LC_MESSAGES/messages.mo | Bin 12085 -> 12085 bytes .../zh_Hans_CN/LC_MESSAGES/messages.mo | Bin 17274 -> 17274 bytes .../zh_Hant_TW/LC_MESSAGES/messages.mo | Bin 15982 -> 15982 bytes 60 files changed, 15 insertions(+), 12 deletions(-) diff --git a/searx/translations/af/LC_MESSAGES/messages.mo b/searx/translations/af/LC_MESSAGES/messages.mo index f1f5847c30498aab5673887c2692d935569efd38..39d5ce36aa29f8318aa46a20d1bcb8857356e90e 100644 GIT binary patch delta 16 XcmbQ|HP359w>Yzrp5f*`@d>;DH3tQ( delta 16 XcmbQ|HP359w>Yz*p26lm@d>;DH2DRp diff --git a/searx/translations/ar/LC_MESSAGES/messages.mo b/searx/translations/ar/LC_MESSAGES/messages.mo index 91186bed09c4ef3e0ba0e070ae0e759199e5db80..d1072568ce56a2ee84cceb6cacb036802287e5cb 100644 GIT binary patch delta 18 acmeypn(_B)#tkak%tm^Ko7J_iXaN9AItJwc delta 18 acmeypn(_B)#tkak%!Yafo7J_iXaN9AEC%BM diff --git a/searx/translations/bg/LC_MESSAGES/messages.mo b/searx/translations/bg/LC_MESSAGES/messages.mo index 4aaedb42017685fc1f7a7e34c86da10dc42f7666..8b77d342f78c72d5d1c34a54ff9c80ea0a6b665a 100644 GIT binary patch delta 18 acmaF3mhs_Q#tmE=%tm^Kn|U>^=m7vmbq0R` delta 18 acmaF3mhs_Q#tmE=%!Yafn|U>^=m7vmX9j%$ diff --git a/searx/translations/bn/LC_MESSAGES/messages.mo b/searx/translations/bn/LC_MESSAGES/messages.mo index 117d597ba7f93eb13d86b9b1e94aa0933d1f5ce0..fd695d78facf8ec5baa619b5e6d3dcc164fbafb1 100644 GIT binary patch delta 24 fcmccL#CX4naRak7tC5}|7sKQS`VyNJr0vxJW}F8= delta 24 fcmccL#CX4naRak7tD&9&7sKQS`VyNJr0vxJW`YMk diff --git a/searx/translations/bo/LC_MESSAGES/messages.mo b/searx/translations/bo/LC_MESSAGES/messages.mo index ece0e471c3132ad2b2fd2a09db379408142c4f69..0af22b4a54d147968f1cc277f58f830c5e0f12c9 100644 GIT binary patch delta 16 XcmcZ?buVgzry#SDp5bO6K_M9cIW7fF delta 16 XcmcZ?buVgzry#STp221xK_M9cIUof~ diff --git a/searx/translations/ca/LC_MESSAGES/messages.mo b/searx/translations/ca/LC_MESSAGES/messages.mo index 8e496509d51e564ea17126a9e6da1bf6e5a2cefe..c5744a3d217d4a2175f1e1b5305896601f8483d8 100644 GIT binary patch delta 18 acmX@n!FZyBal>22lNJpcdz delta 16 YcmdnZy_59I{*Lx diff --git a/searx/translations/el_GR/LC_MESSAGES/messages.mo b/searx/translations/el_GR/LC_MESSAGES/messages.mo index c4662d843388ab0ae800ff326e4491b2ef2923d4..356389398f42109be4c1864c770fbb2611de0ce2 100644 GIT binary patch delta 18 acmbPtf^psn#trwhn2q!dH$T*}wFCf6dIwJc delta 18 acmbPtf^psn#trwhm<{y|Hb2y|wFCf6YzIvM diff --git a/searx/translations/en/LC_MESSAGES/messages.mo b/searx/translations/en/LC_MESSAGES/messages.mo index 18c3608181846c8f8744605d2abeaff2c0520081..c613ebd956e811ca210a4ef2b761d9f7883c4534 100644 GIT binary patch delta 13 UcmdnXyq9^y4n|fZJwq-A03dY(iU0rr delta 13 UcmdnXyq9^y4n|f(Jp(QV03dAxhyVZp diff --git a/searx/translations/eo/LC_MESSAGES/messages.mo b/searx/translations/eo/LC_MESSAGES/messages.mo index 02e3d25e0d991ffc357013967d6cb86d490fe206..89360b7862f25315ede5014315e85b71ab6d971a 100644 GIT binary patch delta 16 XcmaEv^fGBfgFLg5p5f+Z`DcOvK;#C- delta 16 XcmaEv^fGBfgFLgLp26m3`DcOvK-LDt diff --git a/searx/translations/es/LC_MESSAGES/messages.mo b/searx/translations/es/LC_MESSAGES/messages.mo index 432d076ba87071ad632fbe17ed4886d94716e622..1ca44c9ac0b7e8a62b17839c69623f2d76c01133 100644 GIT binary patch delta 18 acmZpk%-B4cal<_=W+Oer%@4H}$pHXELk9T( delta 18 acmZpk%-B4cal<_=W_pA2D<_o{2DtzL diff --git a/searx/translations/fil/LC_MESSAGES/messages.mo b/searx/translations/fil/LC_MESSAGES/messages.mo index b612c5bbdb01efe2a049ae88b25d5c5a13c1765c..9e5033542cc69a262a477010ef747ad9611079bf 100644 GIT binary patch delta 16 XcmaD|@Va0_yCSoZp5f*$#fM@5Lna3J delta 16 XcmaD|@Va0_yCSopp26lW#fM@5Ll_43 diff --git a/searx/translations/fr/LC_MESSAGES/messages.mo b/searx/translations/fr/LC_MESSAGES/messages.mo index e0d547ec71f879cfe96d5673d5330af60699bfff..0088122e10c46d9e4cc0199c8f7a35fc36cbdb94 100644 GIT binary patch delta 18 Ycmdlmg>eHA-PU3@(lgwAPb)|s07zp800000 delta 18 Ycmdlmg>eHA-PU3@)HB$8Pb)|s07z8^{r~^~ diff --git a/searx/translations/gl/LC_MESSAGES/messages.mo b/searx/translations/gl/LC_MESSAGES/messages.mo index 47e74548b3341319ae4bb157897b582f2305c2d8..d96b64aa26824b0e4721e733a486f53df8477a92 100644 GIT binary patch delta 18 acmbQ;%Q(B2al-{IW+Oer&6l-KNCN;x7zZQ( delta 18 acmbQ;%Q(B2al-{IW2O delta 16 XcmZ4Lu+(A0EMaCtJ%i12g*$ivHBSYx diff --git a/searx/translations/id/LC_MESSAGES/messages.mo b/searx/translations/id/LC_MESSAGES/messages.mo index c65b0fb06d5f571d5a6554e7e0fc5bbd6329984d..a3d60506a021fb887299afe57f66183a9b1de43c 100644 GIT binary patch delta 18 ZcmccK!+5=ial->GW+Oer&5yN8qybF*2W9{O delta 18 ZcmccK!+5=ial->GW6LW+Oer%@4Fr$^ZaI>IYx| delta 18 acmZ44&$zOmal>6LW-O9xQ^ delta 18 acmbO`n{n=J#tm1rm<{y|Hec5|rUU>-JqJ$! diff --git a/searx/translations/ko/LC_MESSAGES/messages.mo b/searx/translations/ko/LC_MESSAGES/messages.mo index 81a1f0323733ed1545c9575c053774fbc632a785..93bd609037ca9349d8f573871dbd439557453002 100644 GIT binary patch delta 18 acmX>yf$_)$#tpZ$n2q!dH{aDtmjwVylm{^Y delta 18 acmX>yf$_)$#tpZ$m<{y|Hs94smjwVyh6gVI diff --git a/searx/translations/lt/LC_MESSAGES/messages.mo b/searx/translations/lt/LC_MESSAGES/messages.mo index d2edb6e85069fa76dbb305ade99e1180a8dc4cb1..87a675639f5f67772ce5392462012104ab981f9c 100644 GIT binary patch delta 16 XcmX?Ad8%^50u^Q>J;Tk5Rn#Q_KyU^~ delta 16 XcmX?Ad8%^50u^RMJ%i1QRn#Q_Kw<_) diff --git a/searx/translations/lv/LC_MESSAGES/messages.mo b/searx/translations/lv/LC_MESSAGES/messages.mo index 097080692587168e1a9e74f6fdb12c57ed490c67..7a503a11d8c81f57c361492e47fc391e5dce6ee5 100644 GIT binary patch delta 16 XcmZp1XmQveBEoE>XSi8h!>RFUD-7XSg{@Yymd_E|div delta 16 XcmZp2X>!>RFUD-BXRtX*Yymd_E`|jf diff --git a/searx/translations/nb_NO/LC_MESSAGES/messages.mo b/searx/translations/nb_NO/LC_MESSAGES/messages.mo index 2e38949d1f664d924589a11c9cfb1a5985359ad2..0326251172efdcbe4315abee269cc65181d48d9a 100644 GIT binary patch delta 4357 zcmaLZ3s6*59LMpaXo83$K2wvMh=3w0m>DfmQ)+0IS(a!>YwXwtv9>%4O>X6+j9{8p z_~=Qd#Y|a}PLi6Mjn3E*y_`xX%F4z}D+?OaqVKPJnqD(G>weBX_ul{cpZ`7QE=v}D z;JTm2b$ZnB?`QrG;{OL-boAH1njXe<=5RS)h^y`KR@9}hTi?XEqEg) z#v0QJ7hp8HNL<%c(@>y?k&ju+S0AiL4Y(V1{;2gE9KrEvOvb@Iy}*;OE63O1P`nuj z;=QQr*PsU8hlw4-i}?PW|P4#K{eh?(faJiHrK zfe%sl9YGCr()PEa;+#WOD6+RPF^q5e(9nem7>^^+!6~R26rloFp(?Wib^UYJ7g33A z#VFi`{qX>5z|ZaZlh!uuPJcVPnn`z}YJfykpj6ZV7ome$s2k>B1m20d?=I9ctwIg7 z9Ql~FeCbhaMkZ?-Q59)IUDtvv3v;3m^;ap*(xH+^5>*!t#57FB7@UntENBg*23Uxi z@e=DwRO0JU6RJbq_kum%V$W|!O=M4B>aPplvL~8QGdhIjco;Rn^nTthxCM3nTvTF( zsOu}Nl^EuDF=~cw$nrAh@jNtOf9!~ZP>(3y<(q5Fcud4;s0+)j3s9x6L?yBuHS<~w zVI5wAap&9Bk1Fxas043ARpc&IoC?(aRX7aY)ihM{M&x4-^2MahS9m?1Lj}BcfH9+R z7Anv()BtNyk8l&}S-*;x;{i;?C|0lr9EYmZ6{tru(d&23R2n1cn28Fw1Xa36Pzh~7 z-LMOl=v$~qaLD$zSdXC+J&C&i4EiyOw;BuG6r$qXhdx}2(c1s}XppRFMwR#k4#8He z!2Y}x^afmo8ek2obQ|ynJczR~ez5n#T8QI0egaj&Lr5r-%-&v!6L7ft@ixXc9USkO z2T-L7qGno(8u&p}X{%AYLXee00as zXrhsW5lmY%nu>aM&!8Sj6KWA1K|RASt;bO%J%vi36_fF-JwG(TTN^2;^W#u)e5eGk zN}&GAY#JS!d9Jkp6*z<)u^c0?0u^Wx>il9F5qKIExD9pw9BSKk8tMfegxUoOsQX5vO7BC(pNg8`O;$hZ z`XZOcANvv&xC-^cSZOb)Lj~H5n!#?Ij{8xG4M?=Vf~bTh;jNgDS}VJ;rGqi=qbk{u zm4cw6js(|L=vmaGcph0tW*f3o%*S{UoFo<2T1XZa@)Qe>~ zs?>GZ4Y%Sj+>3n7ackd^Ufe5D_vh$X`+ufAF$XnJ0qWTlBRkWSp>C)}m2M-(;7h2P z?Ld|KZPb0oP%}JfZ9|@>X-7>cDa~6um!f`uOg0UbekMj^9%^PmR6>=g8CKc;^{7Sq zGHQ+N#cudMYJksBkLV{HjnR}h27RasPDaI@F^c-@#(djRWGzR{7&tgGw-dj5p&{ z)Hb>dHSiUvZ8rh+8agWeYDSyD8cn0u#vUhc#gSvhgGOlYbrlG*u_JSOH zd>zKopNo}PguU?uDq)SL0VDa+aW?kBNvMkDA-~M#4xEKms06-1jdvVVwEuskaXlRu zQ2+TDLe2Oy%*L+#n5%@7QTzG@T#wB-7>js~>Ag{nS`(*Gl{t;7=x?aS9GmI=Io}VJ zXa-Ku{?DPI3s;~rdH;`w;k9=Bo=$L*-KlXR)~2gPdial8ws<5{Ho=2|-S zNEV|iwFheC$f&rUQ|M@s6@`%$e-Kc@9u^-l= zDzh1NT_b8D2XG=br_fMI`mxe@NtuDDgr;CF&O>FsC%fT}9u;wJZm7)hhXYP=sMIMd z2^P-|GBSc2-!R?$04{TK5s+2?ufSFl{~j%H~bY&#shd4 zX7>sq3uj_3&PB#e!ZHdDvJ_E9he{qsT?h;7JEIKn@S9X>%{JcC{FBKE`VJ|T?50eCOo zgJ$4$^u6Dr6LrM;qv$v%&K8Qp=pp%dg|Ar3@en24D;6@70Sn))T^ zM9Yz%(8xb-#gj;EVLO_UHniUXc;+8aiXg3s}W-H)iRMFQQv?IibK$=+DzItVTQD9i54$eh#{jnKUAfy&;V>L-JuYPa@OKLC z`Q*YhRSjsQccBw6LQ}g8y(J0O;8t`i4x#;N6&k`_PQ+M<+UfwjV+VIuhIe8rx5x3;zeY;4^6dbFuwme9pKwT|f@nKkr)d z?+Od4aNttR#9DOXF=+cl^qSs`4m<~qbP@XAYBbeNXoOFqfjt*(LHoC%Tecq^_i#eN z55#-1!%1|Yv*<(_LqnL3IhcvF(Ui|a7qkiIVKaI*GKTS8zk<&&X7GHjIQ?yS9vyFB zNt&Ud=t7b*3Pvyno%kkfz;EGF+=*QZ;!r+Im{nkq2S)^LT+H#i@tCSOo#;MpHo6rnyLC1X-yJ2&DZmS~yHteIq47?G28;$5s@%b2F)iy1`ETI9mT48t6~aK!1jA&F@E$e;4p36ftN+{T251O)r=mOqFBlr+q@#kp23s`~K zHEDefdKPX$e*c8|I0qASypNGr!fAAzL8H=4j>6vd8&AQ#n~D?hJ7`2L=!6|;s{VpT z@DcXHQ+OTz7kl8y+O&NV^1rt*1$}P;+J6bUg^lRA>(llmY>pLAV;&7VunF6+FIMsq z@WmU^32%+h>oK3_O=!k;q8a!(&cws$0*3LD6PBZajmBv>7gu`!cgKbzHspF5rl2X@ zg5K+Y;SS8>m---VLq9+r=$WYIvu0*$(UeX?5A$^7{~lom=Hm}=GHyou{TZ{p|DRHD zWv9@U_8FJ%=|J?k4!^$7=;2w4!|^DRBtDVp7K}zSbrX7arbO>SGqeD`4J(l25bnps z6unBpiGG1T??tcQ@30TPg${Tejr6nVSv1nG&|8vKpJt>>G#BmPGg^QypagxdqMrPF zN=H%Q%5R7b4d_Ddh~6FRXQTZWq64o&57#>M>^u?Mo6&%Ni3Rv;bmF72{&@6cJ^A;Z zenEv1Uy2P~CZq%OMkg#lGf{?4JOQu91~fDC(SG-#fjo#e;|BDMyp4QXLI=8_uW$xl zlf#{J`FS7eT?s&3z1(!Fo{?e!Zo{|BNg Bki7r^ diff --git a/searx/translations/nb_NO/LC_MESSAGES/messages.po b/searx/translations/nb_NO/LC_MESSAGES/messages.po index 5e18b8a06..f5870c129 100644 --- a/searx/translations/nb_NO/LC_MESSAGES/messages.po +++ b/searx/translations/nb_NO/LC_MESSAGES/messages.po @@ -11,7 +11,7 @@ msgstr "" "Project-Id-Version: PROJECT VERSION\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" "POT-Creation-Date: 2023-02-20 11:22+0000\n" -"PO-Revision-Date: 2023-02-24 07:07+0000\n" +"PO-Revision-Date: 2023-03-30 12:37+0000\n" "Last-Translator: return42 \n" "Language-Team: Norwegian Bokmål \n" @@ -20,7 +20,7 @@ msgstr "" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" "Plural-Forms: nplurals=2; plural=n != 1;\n" -"X-Generator: Weblate 4.15.2\n" +"X-Generator: Weblate 4.16.4\n" "Generated-By: Babel 2.11.0\n" #. CONSTANT_NAMES['DEFAULT_GROUP_NAME'] @@ -392,10 +392,12 @@ msgid "" "You are using Tor and it looks like you have this external IP address: " "{ip_address}" msgstr "" +"Du bruker Tor og det ser ut som om du har denne eksterne IP adressen: " +"{ip_address}" #: searx/plugins/tor_check.py:86 msgid "You are not using Tor and you have this external IP address: {ip_address}" -msgstr "" +msgstr "Du bruker ikke Tor og du har denne IP adressen: {ip_address}" #: searx/plugins/tracker_url_remover.py:29 msgid "Tracker URL remover" diff --git a/searx/translations/nl/LC_MESSAGES/messages.mo b/searx/translations/nl/LC_MESSAGES/messages.mo index fb0193b085d391a945d7a86f9ff308f0d487bdda..2982f47a1ba042042b7c2b5e0925f07253ed002f 100644 GIT binary patch delta 18 ZcmccL%Xq(+al-{IW+Oer&6l-!WB^QM2Jrv@ delta 18 ZcmccL%Xq(+al-{IW diff --git a/searx/translations/oc/LC_MESSAGES/messages.mo b/searx/translations/oc/LC_MESSAGES/messages.mo index 339debbab87164cbe11dcb682b96158295d53888..e479a21f6cabca7c46f94116757c8aa9ef1722f2 100644 GIT binary patch delta 1092 zcmXZbT}V@59LMp0xGl|XmM_^dYeUHw%-YD1#)4V&LIqMpkWmOih0Db(3?dG_fzS(r zqq{DGE`nGKRH6_@-IP#KP_&yav1w5SQaqY{L@rsV(A4 zTxu}Kswsq4o2|eIX0a95U>-F=0eNhO7vI_&tic70<4;UrYoj~}5f{EPaqmS}VmZOB^IgK9s3eK?5PzytJQ5jB1qHP1`b zMn8G&^QaU5?oiOezflqW@#_8vlTfe1wfGD*?jzpBGS=Y`o$?~8|CZ+jGL_}A2@9wb znnMM&gxAoip;wVkdfrFvq==gMIcD%3_F{4!X~Ofk9`B&W6;OBL1u8YKu?Nc-#Bj5l z^CpZ^??ON8+d&G-)nU}YejVLJ|Z_3K{!E-GgwRI27sxA=qS0_tdgdHsQ?J8u{j zPz1HXX4D<&#DGI#GX({(4Yfepb3ZDeBd8NOh1$_&&)cZ}dtSYW3aEq%@I4M=8TDK4 z;$PL}9o9da&SbJ8e98Ddk%<&^LZA?-LF()h%O8i)aP$3~fi9v>G(2$2)IEcG=1Y;%s2NK9*qrB*{w|EF=F^Y?L26s^l)|Z;)U=u2li>QUWup0YN z<3>t7KjJYOGHLjVJT}Km6Mvy5ZlZQ#4_U+Vh)(TQ*pBB=2|U68CQ;)@P~UllN_5J# z&!Bevn@2$l|3F3b)76*p6ZKU*iqFdYag+FjdK$ynOsBkp`p#`Bj_@yLvA%_l`x%GvFddaxhmEd&!_@~+XO==8)i^4n56)TC)_!;WE2y1ZM+LNn zN^l2tNB(2BMBmc#clqYaJISBsiL`z{%1;s5{u diff --git a/searx/translations/pa/LC_MESSAGES/messages.mo b/searx/translations/pa/LC_MESSAGES/messages.mo index f0ffbda43310b33724ea5c4bb73b4fdbcea1ff1d..619a237c92187b071c79ae52c8ad0b68c0369020 100644 GIT binary patch delta 13 UcmdnUypegs8b($lJwq-A03P)NW&i*H delta 13 UcmdnUypegs8b($_Jp(QV03PiFWB>pF diff --git a/searx/translations/pap/LC_MESSAGES/messages.mo b/searx/translations/pap/LC_MESSAGES/messages.mo index b97570543f0078dcba364e3b54abafd306768e89..c3e9e4e4a8555b3bc2e456fd6cadf145557b3b2b 100644 GIT binary patch delta 16 XcmZ1|uux#bZB}L@J;TlSSe=;wGAadP delta 16 XcmZ1|uux#bZB}MOJ%i2nSe=;wG8_e9 diff --git a/searx/translations/pl/LC_MESSAGES/messages.mo b/searx/translations/pl/LC_MESSAGES/messages.mo index 978fa553da6cf34faf759ac26e6651b1dfac498d..ae5602472b4713638e2719db4e76966da9c05a12 100644 GIT binary patch delta 18 ZcmeC4#MnKFaYKeSvyq5al;2KW+Oer&7ZYoWC28T2B-i4 delta 18 ZcmeC5z}P>5al;2KW8<#toU;%tm^Kn{%`q4FFC{2Oa8<#toU;%!Yafn{%`q4FFC(2OIzZ diff --git a/searx/translations/si/LC_MESSAGES/messages.mo b/searx/translations/si/LC_MESSAGES/messages.mo index c5535a16cedb830f7ff078e33ec6aa05e4e1b45d..0e9ab378786b92c7a9a0024ec232e4639033bd8c 100644 GIT binary patch delta 16 Xcmdm}w^46HJvXzFp5f*u?)BUNGaUto delta 16 Xcmdm}w^46HJvXzVp26lO?)BUNGYg1W+Oer&F3^)Bmqh(2P^;p delta 18 ZcmccA!g#5Lal>g1WW1QUFN<2OIzZ delta 18 ZcmaFf#`wIAaYMNVv!R~B<|>W1QUFNx2O0nX diff --git a/searx/translations/ta/LC_MESSAGES/messages.mo b/searx/translations/ta/LC_MESSAGES/messages.mo index cb11269d89d100a8481c88c6feb5fe8f180d6155..81fe927439c8cdca710bae0aa9d02d74a7ff2829 100644 GIT binary patch delta 1958 zcmXZce@xVM7{Kv|pxF--;l_m%W&VSO`c8n~^JPmbX)?6*OR<^d5o6FT!)`DtQ3!K&a+2_j`ug~ZE6p;@-A}`<A2XmOeTqKqpWw4(MJ2C7vFMfnB=2tL+H?a@vibZzedGzC| z5)<%we4PaLU_00IN=2%05c!pp{LzhELm$rKO7xalBEK+rw#;PI#NE}*W5{OYUDO?2 z#t=^92J{eJ=l$r$cW@hijxS=>a*^FQiT7YlK%@m5(1~OC08Uv9bb&vy3!UXGjzhL@ zqGo;pwdCL8{g_o@j%Y1vf^9e-`>`8$p*}y2=~x&PS%xK;fz3!!#oELmi@{F&fqkf@ zJCAudi&R^jY*Q<4b*0xsS)`Z9jipv<1Ng` z##-~WY{xas592%%`w^`nF3hkRjVs-cn$jLbO(2Q%n_R>X@NXoB9AlL{{0&=h{#vtv zE_5@uun>=+R^(IEroMu@-YwKbme)~#4b)#}b}))tnV&_DQ}Q2Ux7dQiWITy__O#>O78KEXFT!G0vhFb2f-<$7+m_*aS{8zeyI_3n^=(DbXLOfitM5 zcJ&g}Qu**sd>V(b+rIx7YM|SnG>N!TOI(Rs`Zn8FP)H`S-(+j z&1sZ7aQ>mqO(-TYT-Pv~n5)&QwPI^)xn@smwu+wJB+AzNv+s{FUf=KU;rso3KcCN! zSEgLUQ?7IC=GpJMMCQCAw~5F_zsQsL31(vbVv$_jfd#l9eK?3K@jSkPzu`@+GgK&& z$9!ywdGAB4V*VZ0qd#Ew*^N2Op9+XAXBi~9(1&Z5ninsjm-%Ik;v8C78x(mKr!a(T z?=&0k!+rudircxKS0qx72asP$@kbBxHJ0EkuE9XDE%FnCc(I9SBX`#^e-t^4jG~@s z8q077o6yJZI=9e=XK@#PjE8XbGLgeLfeWxYEV2a~@OB)=6?nmBpbPwkk6=!OG~pAD zCs8AR8#U!$;N9pcH6>b)8el8liWYX^GpNtcpc@O9i;x@%Vj6Bjk}7r!gA5(uU3d&N zbr&!nuOr!(Y>w-}O4NA+>IS{|B91%rDx%{R>BUSO#YK40@hivwkicyzAP?Gn<){nS zV-6-z7dVW1f|JhtH0nuHsPA3Em#FX06DNm{CK0@gH1- z4fmR_We=`rehTLi*!O53U#g81`?{%`ZKWV%n1f?C-X_9oKkSV$ivu-M~HX=we~d+ zh%CfLjA9pF!3o^Na|2vt5BY&~_2M$>sF`^cH{-EJksiF-Xqy^t+h|VIj@ko4VoX*Yuk z)Qzf97pg~HXgBIcN$37JYV9YT>z_ORiJ4qaZ#B>RPgwW-@N7Y9(+9m9qC{eQ+l zBl-pP0ryrDSw3p5TTnOLi5gfRF2*6$rv1S2x?^CQd9E3YxPQQL2(?7hj(=mB{;TqD zHxajB88u3vmSALu$X@t9JN~{{b+b5o`be diff --git a/searx/translations/ta/LC_MESSAGES/messages.po b/searx/translations/ta/LC_MESSAGES/messages.po index 0fda30744..401f4af02 100644 --- a/searx/translations/ta/LC_MESSAGES/messages.po +++ b/searx/translations/ta/LC_MESSAGES/messages.po @@ -10,20 +10,22 @@ # POORAJITH ST , 2019 # Prasanna Venkadesh , 2019 # Markus Heiser , 2022. +# return42 , 2023. msgid "" msgstr "" -"Project-Id-Version: searx\n" +"Project-Id-Version: searx\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" "POT-Creation-Date: 2023-02-20 11:22+0000\n" -"PO-Revision-Date: 2022-11-04 07:18+0000\n" -"Last-Translator: Markus Heiser \n" +"PO-Revision-Date: 2023-03-30 12:37+0000\n" +"Last-Translator: return42 \n" +"Language-Team: Tamil \n" "Language: ta\n" -"Language-Team: Tamil " -"\n" -"Plural-Forms: nplurals=2; plural=n != 1;\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=2; plural=n != 1;\n" +"X-Generator: Weblate 4.16.4\n" "Generated-By: Babel 2.11.0\n" #. CONSTANT_NAMES['DEFAULT_GROUP_NAME'] @@ -427,7 +429,7 @@ msgstr "பக்கம் கிடைக்கவில்லை" #: searx/templates/simple/404.html:6 #, python-format msgid "Go to %(search_page)s." -msgstr "%(search_page)s-க்கு செல்" +msgstr "%(search_page)s-க்கு செல்." #: searx/templates/simple/404.html:6 msgid "search page" @@ -956,7 +958,7 @@ msgstr "முன் பக்கத்தைக் காட்டு" #: searx/templates/simple/search.html:9 #: searx/templates/simple/simple_search.html:5 msgid "Search for..." -msgstr "எதைப்பற்றி தேட வேண்டும்?" +msgstr "எதைப்பற்றி தேட வேண்டும..." #: searx/templates/simple/search.html:10 #: searx/templates/simple/simple_search.html:6 @@ -1529,4 +1531,3 @@ msgstr "காணொளிகளை மறை" #~ msgid "Automatically detect the query search language and switch to it." #~ msgstr "" - diff --git a/searx/translations/te/LC_MESSAGES/messages.mo b/searx/translations/te/LC_MESSAGES/messages.mo index c4149752c932e7b9fe2d26b868e7ab0614754279..d952dd94fc8dbd2e9f7f8bce74802f9852747859 100644 GIT binary patch delta 18 acmcbxneoDA#tr<+%tm^Kn}wA3nE?Pq56VH7RBzJ;Tj6rD8+?J`@IV delta 16 XcmdlQw>56VH7RC8J%i0RrD8+?J_ZJF diff --git a/searx/translations/zh_Hans_CN/LC_MESSAGES/messages.mo b/searx/translations/zh_Hans_CN/LC_MESSAGES/messages.mo index ddd626e5f7c4a44cee0540b025d3570cbcf91e78..87d78f394b184d20a0f2bd9e2b1b489b60f4b7b8 100644 GIT binary patch delta 18 Zcmey>#`vp^al>mZW+Oer&F{2&BmhyE2g?8e delta 18 Zcmey>#`vp^al>mZW Date: Fri, 31 Mar 2023 07:56:50 +0000 Subject: [PATCH 04/14] Bump redis from 4.5.1 to 4.5.4 Bumps [redis](https://github.com/redis/redis-py) from 4.5.1 to 4.5.4. - [Release notes](https://github.com/redis/redis-py/releases) - [Changelog](https://github.com/redis/redis-py/blob/master/CHANGES) - [Commits](https://github.com/redis/redis-py/compare/v4.5.1...v4.5.4) --- updated-dependencies: - dependency-name: redis dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3ee417d7c..f6b46ec8a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ Brotli==1.0.9 uvloop==0.17.0 httpx-socks[asyncio]==0.7.2 setproctitle==1.3.2 -redis==4.5.1 +redis==4.5.4 markdown-it-py==2.2.0 typing_extensions==4.5.0 fasttext-predict==0.9.2.1 From 615c8ebcfdd9e25ab5868a6501033487a3542ccd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 31 Mar 2023 07:56:56 +0000 Subject: [PATCH 05/14] Bump selenium from 4.8.2 to 4.8.3 Bumps [selenium](https://github.com/SeleniumHQ/Selenium) from 4.8.2 to 4.8.3. - [Release notes](https://github.com/SeleniumHQ/Selenium/releases) - [Commits](https://github.com/SeleniumHQ/Selenium/commits) --- updated-dependencies: - dependency-name: selenium dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index e7fea88d2..c1d93b764 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,7 +4,7 @@ cov-core==1.15.0 black==22.12.0 pylint==2.17.1 splinter==0.19.0 -selenium==4.8.2 +selenium==4.8.3 twine==4.0.2 Pallets-Sphinx-Themes==2.0.3 Sphinx==5.3.0 From b99e028ed08144e51b1cd474330e97a0d3e5c57e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 31 Mar 2023 11:39:17 +0000 Subject: [PATCH 06/14] Bump sharp from 0.31.3 to 0.32.0 in /searx/static/themes/simple Bumps [sharp](https://github.com/lovell/sharp) from 0.31.3 to 0.32.0. - [Release notes](https://github.com/lovell/sharp/releases) - [Changelog](https://github.com/lovell/sharp/blob/main/docs/changelog.md) - [Commits](https://github.com/lovell/sharp/compare/v0.31.3...v0.32.0) --- updated-dependencies: - dependency-name: sharp dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- searx/static/themes/simple/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/searx/static/themes/simple/package.json b/searx/static/themes/simple/package.json index 4c1cd169a..fdb4f88bf 100644 --- a/searx/static/themes/simple/package.json +++ b/searx/static/themes/simple/package.json @@ -14,7 +14,7 @@ "ionicons": "^7.1.0", "less": "^4.1.3", "less-plugin-clean-css": "^1.5.1", - "sharp": "^0.31.0", + "sharp": "^0.32.0", "stylelint": "^13.13.1", "stylelint-config-standard": "^22.0.0", "ejs": "^3.1.8", From c8d78355ff41a24174ce6d7b7e24309dec0d7c3a Mon Sep 17 00:00:00 2001 From: Venca24 Date: Fri, 31 Mar 2023 16:11:27 +0200 Subject: [PATCH 07/14] [fix] engine seznam --- searx/engines/seznam.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/searx/engines/seznam.py b/searx/engines/seznam.py index 48a167ce0..5588c230f 100644 --- a/searx/engines/seznam.py +++ b/searx/engines/seznam.py @@ -30,7 +30,8 @@ base_url = 'https://search.seznam.cz/' def request(query, params): - response_index = get(base_url, headers=params['headers'], raise_for_httperror=True) + response_index = get( + base_url, headers=params['headers'], raise_for_httperror=True) dom = html.fromstring(response_index.text) url_params = { @@ -54,8 +55,9 @@ def response(resp): results = [] dom = html.fromstring(resp.content.decode()) - for result_element in eval_xpath_list(dom, '//div[@data-dot="results"]/div'): - result_data = eval_xpath_getindex(result_element, './/div[contains(@class, "bec586")]', 0, default=None) + for result_element in eval_xpath_list(dom, '//div[@id="searchpage-root"]//div[@class="Layout--left"]/div[@class="f2c528"]'): + result_data = eval_xpath_getindex( + result_element, './/div[@class="c8774a" or @class="e69e8d a11657"]', 0, default=None) if result_data is None: continue title_element = eval_xpath_getindex(result_element, './/h3/a', 0) @@ -63,7 +65,7 @@ def response(resp): { 'url': title_element.get('href'), 'title': extract_text(title_element), - 'content': extract_text(eval_xpath(result_data, './/div[@class="_3eded7"]')), + 'content': extract_text(result_data), } ) From 509afbbb846b98094a9b596b1abb4e193092a3cb Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Fri, 31 Mar 2023 17:25:39 +0200 Subject: [PATCH 08/14] [fix] engine seznam: fix issues reported by black & pylint Signed-off-by: Markus Heiser --- searx/engines/seznam.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/searx/engines/seznam.py b/searx/engines/seznam.py index 5588c230f..36a38848a 100644 --- a/searx/engines/seznam.py +++ b/searx/engines/seznam.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: AGPL-3.0-or-later -""" - Seznam +# lint: pylint +"""Seznam + """ from urllib.parse import urlencode @@ -11,7 +12,6 @@ from searx.utils import ( extract_text, eval_xpath_list, eval_xpath_getindex, - eval_xpath, ) # about @@ -30,8 +30,7 @@ base_url = 'https://search.seznam.cz/' def request(query, params): - response_index = get( - base_url, headers=params['headers'], raise_for_httperror=True) + response_index = get(base_url, headers=params['headers'], raise_for_httperror=True) dom = html.fromstring(response_index.text) url_params = { @@ -55,9 +54,12 @@ def response(resp): results = [] dom = html.fromstring(resp.content.decode()) - for result_element in eval_xpath_list(dom, '//div[@id="searchpage-root"]//div[@class="Layout--left"]/div[@class="f2c528"]'): + for result_element in eval_xpath_list( + dom, '//div[@id="searchpage-root"]//div[@class="Layout--left"]/div[@class="f2c528"]' + ): result_data = eval_xpath_getindex( - result_element, './/div[@class="c8774a" or @class="e69e8d a11657"]', 0, default=None) + result_element, './/div[@class="c8774a" or @class="e69e8d a11657"]', 0, default=None + ) if result_data is None: continue title_element = eval_xpath_getindex(result_element, './/h3/a', 0) From afd8fcce36dc03c4ca0ff5878938b2b50459a924 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Sun, 12 Feb 2023 12:14:15 +0100 Subject: [PATCH 09/14] [mod] plugin limiter: improve the log messages In debug mode more detailed logging is needed to evaluate if an access should have been blocked by the limiter. BTW: remove duplicate code checking bot signature ``re_bot.match(user_agent)`` Signed-off-by: Markus Heiser --- searx/plugins/limiter.py | 39 ++++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/searx/plugins/limiter.py b/searx/plugins/limiter.py index b66a0805c..baf2c1726 100644 --- a/searx/plugins/limiter.py +++ b/searx/plugins/limiter.py @@ -17,13 +17,14 @@ import re from flask import request from searx import redisdb +from searx.plugins import logger from searx.redislib import incr_sliding_window name = "Request limiter" description = "Limit the number of request" default_on = False preference_section = 'service' - +logger = logger.getChild('limiter') re_bot = re.compile( r'(' @@ -42,44 +43,52 @@ def is_accepted_request() -> bool: user_agent = request.headers.get('User-Agent', '') x_forwarded_for = request.headers.get('X-Forwarded-For', '') - if request.path == '/image_proxy': - if re_bot.match(user_agent): - return False - return True + if re_bot.match(user_agent): + logger.debug("BLOCK %s: detected bot", x_forwarded_for) + return False if request.path == '/search': c_burst = incr_sliding_window(redis_client, 'IP limit, burst' + x_forwarded_for, 20) c_10min = incr_sliding_window(redis_client, 'IP limit, 10 minutes' + x_forwarded_for, 600) if c_burst > 15 or c_10min > 150: - logger.debug("to many request") # pylint: disable=undefined-variable - return False - - if re_bot.match(user_agent): - logger.debug("detected bot") # pylint: disable=undefined-variable + logger.debug("BLOCK %s: to many request", x_forwarded_for) return False if len(request.headers.get('Accept-Language', '').strip()) == '': - logger.debug("missing Accept-Language") # pylint: disable=undefined-variable + logger.debug("BLOCK %s: missing Accept-Language", x_forwarded_for) return False if request.headers.get('Connection') == 'close': - logger.debug("got Connection=close") # pylint: disable=undefined-variable + logger.debug("BLOCK %s: got Connection=close", x_forwarded_for) return False accept_encoding_list = [l.strip() for l in request.headers.get('Accept-Encoding', '').split(',')] if 'gzip' not in accept_encoding_list and 'deflate' not in accept_encoding_list: - logger.debug("suspicious Accept-Encoding") # pylint: disable=undefined-variable + logger.debug("BLOCK %s: suspicious Accept-Encoding", x_forwarded_for) return False if 'text/html' not in request.accept_mimetypes: - logger.debug("Accept-Encoding misses text/html") # pylint: disable=undefined-variable + logger.debug("BLOCK %s: Accept-Encoding misses text/html", x_forwarded_for) return False if request.args.get('format', 'html') != 'html': c = incr_sliding_window(redis_client, 'API limit' + x_forwarded_for, 3600) if c > 4: - logger.debug("API limit exceeded") # pylint: disable=undefined-variable + logger.debug("BLOCK %s: API limit exceeded", x_forwarded_for) return False + + logger.debug( + "OK %s: '%s'" % (x_forwarded_for, request.path) + + " || form: %s" % request.form + + " || Accept: %s" % request.headers.get('Accept', '') + + " || Accept-Language: %s" % request.headers.get('Accept-Language', '') + + " || Accept-Encoding: %s" % request.headers.get('Accept-Encoding', '') + + " || Content-Type: %s" % request.headers.get('Content-Type', '') + + " || Content-Length: %s" % request.headers.get('Content-Length', '') + + " || Connection: %s" % request.headers.get('Connection', '') + + " || User-Agent: %s" % user_agent + ) + return True From 8de8070ed9c8c8423521ba722e850f1538ee0b1d Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Sat, 1 Apr 2023 19:33:13 +0200 Subject: [PATCH 10/14] [fix] engine google-News: fix decoding of URLs Google-News returns internal links where the origin URL is encoded in a base64 (RFC 2045 aka URL-safe) string. Closes: https://github.com/searxng/searxng/issues/1959 Signed-off-by: Markus Heiser --- searx/engines/google_news.py | 41 ++++++++++-------------------------- 1 file changed, 11 insertions(+), 30 deletions(-) diff --git a/searx/engines/google_news.py b/searx/engines/google_news.py index ae55ca9cb..2cee2e2c1 100644 --- a/searx/engines/google_news.py +++ b/searx/engines/google_news.py @@ -27,10 +27,8 @@ The google news API ignores some parameters from the common :ref:`google API`: from typing import TYPE_CHECKING -import binascii -import re from urllib.parse import urlencode -from base64 import b64decode +import base64 from lxml import html import babel @@ -144,34 +142,17 @@ def response(resp): for result in eval_xpath_list(dom, '//div[@class="xrnccd"]'): - # The first tag in the
contains the link to the - # article The href attribute of the is a google internal link, - # we can't use. The real link is hidden in the jslog attribute: - # - # + # The first tag in the
contains the link to the article + # The href attribute of the tag is a google internal link, we have + # to decode - jslog = eval_xpath_getindex(result, './article/a/@jslog', 0) - url = re.findall('http[^;]*', jslog) - if url: - url = url[0] - else: - # The real URL is base64 encoded in the json attribute: - # jslog="95014; 5:W251bGwsbnVsbCxudW...giXQ==; track:click" - jslog = jslog.split(";")[1].split(':')[1].strip() - try: - padding = (4 - (len(jslog) % 4)) * "=" - jslog = b64decode(jslog + padding) - except binascii.Error: - # URL can't be read, skip this result - continue + href = eval_xpath_getindex(result, './article/a/@href', 0) + href = href.split('?')[0] + href = href.split('/')[-1] + href = base64.urlsafe_b64decode(href + '====') + href = href[4:].split(b'\xd2')[0] + href = href.decode() - # now we have : b'[null, ... null,"https://www.cnn.com/.../index.html"]' - url = re.findall('http[^;"]*', str(jslog))[0] - - # the first

tag in the
contains the title of the link title = extract_text(eval_xpath(result, './article/h3[1]')) # The pub_date is mostly a string like 'yesertday', not a real @@ -189,7 +170,7 @@ def response(resp): results.append( { - 'url': url, + 'url': href, 'title': title, 'content': content, 'img_src': img_src, From 66810ce71122183f52446cb110dc76e4cae9b7ba Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Sat, 1 Apr 2023 12:34:58 +0200 Subject: [PATCH 11/14] [mod] limiter: minor improvements - requests without HTTP header 'Connection' or missing 'User-Agent' will be blocked by the limiter - re_bot is related to 'User-Agent' and has been renamed to block_user_agent Signed-off-by: Markus Heiser --- searx/plugins/limiter.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/searx/plugins/limiter.py b/searx/plugins/limiter.py index baf2c1726..e1cd0e408 100644 --- a/searx/plugins/limiter.py +++ b/searx/plugins/limiter.py @@ -26,13 +26,17 @@ default_on = False preference_section = 'service' logger = logger.getChild('limiter') -re_bot = re.compile( +block_user_agent = re.compile( r'(' - + r'[Cc][Uu][Rr][Ll]|[wW]get|Scrapy|splash|JavaFX|FeedFetcher|python-requests|Go-http-client|Java|Jakarta|okhttp' + + r'unknown' + + r'|[Cc][Uu][Rr][Ll]|[wW]get|Scrapy|splash|JavaFX|FeedFetcher|python-requests|Go-http-client|Java|Jakarta|okhttp' + r'|HttpClient|Jersey|Python|libwww-perl|Ruby|SynHttpClient|UniversalFeedParser|Googlebot|GoogleImageProxy' + r'|bingbot|Baiduspider|yacybot|YandexMobileBot|YandexBot|Yahoo! Slurp|MJ12bot|AhrefsBot|archive.org_bot|msnbot' + r'|MJ12bot|SeznamBot|linkdexbot|Netvibes|SMTBot|zgrab|James BOT|Sogou|Abonti|Pixray|Spinn3r|SemrushBot|Exabot' + r'|ZmEu|BLEXBot|bitlybot' + # when you block requests from Farside instances, your instance will + # disappear from https://farside.link/ + # + r'|Farside' + r')' ) @@ -40,14 +44,15 @@ re_bot = re.compile( def is_accepted_request() -> bool: # pylint: disable=too-many-return-statements redis_client = redisdb.client() - user_agent = request.headers.get('User-Agent', '') + user_agent = request.headers.get('User-Agent', 'unknown') x_forwarded_for = request.headers.get('X-Forwarded-For', '') - if re_bot.match(user_agent): - logger.debug("BLOCK %s: detected bot", x_forwarded_for) + if block_user_agent.match(user_agent): + logger.debug("BLOCK %s: %s --> detected User-Agent: %s" % (x_forwarded_for, request.path, user_agent)) return False if request.path == '/search': + c_burst = incr_sliding_window(redis_client, 'IP limit, burst' + x_forwarded_for, 20) c_10min = incr_sliding_window(redis_client, 'IP limit, 10 minutes' + x_forwarded_for, 600) if c_burst > 15 or c_10min > 150: From 0430662189676be3807099fca992bd318fec3dc1 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Sun, 2 Apr 2023 18:35:56 +0200 Subject: [PATCH 12/14] [fix] engine google-News: fix decoding of URLs (part 2) Follow up of 8de8070ed to fix the issue reported by AlyoshaVasilieva [1]. [1] https://github.com/searxng/searxng/issues/1959#issuecomment-1493300574 Signed-off-by: Markus Heiser --- searx/engines/google_news.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/searx/engines/google_news.py b/searx/engines/google_news.py index 2cee2e2c1..4b1bffa30 100644 --- a/searx/engines/google_news.py +++ b/searx/engines/google_news.py @@ -150,7 +150,7 @@ def response(resp): href = href.split('?')[0] href = href.split('/')[-1] href = base64.urlsafe_b64decode(href + '====') - href = href[4:].split(b'\xd2')[0] + href = href[href.index(b'http') :].split(b'\xd2')[0] href = href.decode() title = extract_text(eval_xpath(result, './article/h3[1]')) From a762172bf748f6d4d590faaf9df50e778ebf09ba Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Mon, 3 Apr 2023 09:52:16 +0200 Subject: [PATCH 13/14] [fix] engine ddg: quote !bangs in a request send to ddg Closes: https://github.com/searxng/searxng/issues/392 Signed-off-by: Markus Heiser --- searx/engines/duckduckgo.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/searx/engines/duckduckgo.py b/searx/engines/duckduckgo.py index 85e977bdb..4dd23c759 100644 --- a/searx/engines/duckduckgo.py +++ b/searx/engines/duckduckgo.py @@ -6,6 +6,7 @@ DuckDuckGo Lite """ from typing import TYPE_CHECKING +import re from urllib.parse import urlencode import json import babel @@ -15,6 +16,7 @@ from searx import ( network, locales, redislib, + external_bang, ) from searx import redisdb from searx.utils import ( @@ -197,6 +199,17 @@ ddg_lang_map = { def request(query, params): + # quote ddg bangs + query_parts = [] + # for val in re.split(r'(\s+)', query): + for val in re.split(r'(\s+)', query): + if not val.strip(): + continue + if val.startswith('!') and external_bang.get_node(external_bang.EXTERNAL_BANGS, val[1:]): + val = f"'{val}'" + query_parts.append(val) + query = ' '.join(query_parts) + eng_region = traits.get_region(params['searxng_locale'], traits.all_locale) # eng_lang = get_ddg_lang(traits, params['searxng_locale']) From 03f94962b63eb8e189b0b4983c0329d0c29c34ab Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Mon, 3 Apr 2023 19:36:28 +0200 Subject: [PATCH 14/14] [fix] limiter: never block a /healthz request Related: https://github.com/searxng/searxng/issues/2310#issuecomment-1494417531 Signed-off-by: Markus Heiser --- searx/plugins/limiter.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/searx/plugins/limiter.py b/searx/plugins/limiter.py index e1cd0e408..2a9e6f8f5 100644 --- a/searx/plugins/limiter.py +++ b/searx/plugins/limiter.py @@ -47,6 +47,9 @@ def is_accepted_request() -> bool: user_agent = request.headers.get('User-Agent', 'unknown') x_forwarded_for = request.headers.get('X-Forwarded-For', '') + if request.path == '/healthz': + return True + if block_user_agent.match(user_agent): logger.debug("BLOCK %s: %s --> detected User-Agent: %s" % (x_forwarded_for, request.path, user_agent)) return False