Skip to content

Commit

Permalink
Simplify Economist [skip ci]
Browse files Browse the repository at this point in the history
  • Loading branch information
ping committed Aug 9, 2023
1 parent 7529d23 commit 81b506e
Showing 1 changed file with 7 additions and 16 deletions.
23 changes: 7 additions & 16 deletions recipes/economist.recipe.py
Original file line number Diff line number Diff line change
Expand Up @@ -256,6 +256,8 @@ def __init__(self, *args, **kwargs):
self.log.warn(
"Kindle Output profile being used, reducing image quality to keep file size below amazon email threshold"
)

def get_browser(self):
br = BasicNewsRecipe.get_browser(self)
# Add a cookie indicating we have accepted Economist's cookie
# policy (needed when running from some European countries)
Expand All @@ -280,25 +282,14 @@ def __init__(self, *args, **kwargs):
)
br.cookiejar.set_cookie(ck)
br.set_handle_gzip(True)
self._br = br

# We send no cookies to avoid triggering bot detection
def get_browser(self, *args, **kwargs):
return self

def clone_browser(self, *args, **kwargs):
return self.get_browser()
return br

def open_novisit(self, *args, **kwargs):
target_url = args[0]
p, ext = splitext(urlparse(target_url).path)
def get_url_specific_delay(self, url):
p, ext = splitext(urlparse(url).path)
if not ext:
# not an asset, e.g. .png .jpg
time.sleep(random.choice([r for r in range(1, 3)]))

return self._br.open_novisit(*args, **kwargs)

open = open_novisit
return random.choice([r for r in range(1, 3)])
return 0

def preprocess_raw_html(self, raw, _):
root = parse(raw)
Expand Down

0 comments on commit 81b506e

Please sign in to comment.