Skip to content

Commit

Permalink
Revert "Simplify Economist [skip ci]"
Browse files Browse the repository at this point in the history
This reverts commit 81b506e.
  • Loading branch information
ping committed Aug 9, 2023
1 parent 81b506e commit 9c26ac3
Showing 1 changed file with 16 additions and 7 deletions.
23 changes: 16 additions & 7 deletions recipes/economist.recipe.py
Original file line number Diff line number Diff line change
Expand Up @@ -256,8 +256,6 @@ def __init__(self, *args, **kwargs):
self.log.warn(
"Kindle Output profile being used, reducing image quality to keep file size below amazon email threshold"
)

def get_browser(self):
br = BasicNewsRecipe.get_browser(self)
# Add a cookie indicating we have accepted Economist's cookie
# policy (needed when running from some European countries)
Expand All @@ -282,14 +280,25 @@ def get_browser(self):
)
br.cookiejar.set_cookie(ck)
br.set_handle_gzip(True)
return br
self._br = br

# We send no cookies to avoid triggering bot detection
def get_browser(self, *args, **kwargs):
return self

def clone_browser(self, *args, **kwargs):
return self.get_browser()

def get_url_specific_delay(self, url):
p, ext = splitext(urlparse(url).path)
def open_novisit(self, *args, **kwargs):
target_url = args[0]
p, ext = splitext(urlparse(target_url).path)
if not ext:
# not an asset, e.g. .png .jpg
return random.choice([r for r in range(1, 3)])
return 0
time.sleep(random.choice([r for r in range(1, 3)]))

return self._br.open_novisit(*args, **kwargs)

open = open_novisit

def preprocess_raw_html(self, raw, _):
root = parse(raw)
Expand Down

0 comments on commit 9c26ac3

Please sign in to comment.