From 290090d9ff91d5651aa9627cc5bc7394e72994e6 Mon Sep 17 00:00:00 2001 From: ping Date: Thu, 12 Oct 2023 19:01:53 +0800 Subject: [PATCH] Fix request_as_gbot [skip ci] --- recipes/includes/recipes_shared.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/recipes/includes/recipes_shared.py b/recipes/includes/recipes_shared.py index 4760cfa0..38b892de 100644 --- a/recipes/includes/recipes_shared.py +++ b/recipes/includes/recipes_shared.py @@ -304,11 +304,16 @@ def clone_browser(self, *args, **kwargs): return self.get_browser() def open_novisit(self, *args, **kwargs): + br = browser() if self.request_as_gbot: - br = browser( - user_agent="Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" - ) - br.addheaders = [("referer", "https://www.google.com/")] + br.addheaders = [ + ( + "User-agent", + "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", + ), + ("Referer", "https://www.google.com/"), + ("X-Forwarded-For", "66.249.66.1"), + ] else: br = browser() br.set_handle_gzip(True)