From 0a83be0ec92928bb07e19a28ff009d624b307030 Mon Sep 17 00:00:00 2001 From: dalf Date: Thu, 22 Jan 2015 11:40:28 +0100 Subject: [PATCH] [fix] google engine: depending on the IP of the searx instance, each searx request where making two HTTP requests (see https://support.google.com/websearch/answer/873?hl=en ) --- searx/engines/google.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/searx/engines/google.py b/searx/engines/google.py index d4db82a2e..9c768260a 100644 --- a/searx/engines/google.py +++ b/searx/engines/google.py @@ -11,6 +11,7 @@ from urllib import urlencode from urlparse import urlparse, parse_qsl from lxml import html +from searx.poolrequests import get from searx.engines.xpath import extract_text, extract_url # engine dependent config @@ -39,6 +40,17 @@ images_xpath = './/div/a' image_url_xpath = './@href' image_img_src_xpath = './img/@src' +pref_cookie = '' + + +# see https://support.google.com/websearch/answer/873?hl=en +def get_google_pref_cookie(): + global pref_cookie + if pref_cookie == '': + resp = get('https://www.google.com/ncr', allow_redirects=False) + pref_cookie = resp.cookies["PREF"] + return pref_cookie + # remove google-specific tracking-url def parse_url(url_string): @@ -64,6 +76,7 @@ def request(query, params): query=urlencode({'q': query})) params['headers']['Accept-Language'] = language + params['cookies']['PREF'] = get_google_pref_cookie() return params