with actual byline element
byline = soup.select_one("ul.as")
diff --git a/recipes/thediplomat.recipe.py b/recipes/thediplomat.recipe.py
index 6044c9b3..38985451 100644
--- a/recipes/thediplomat.recipe.py
+++ b/recipes/thediplomat.recipe.py
@@ -15,7 +15,6 @@
sys.path.append(os.environ["recipes_includes"])
from recipes_shared import WordPressNewsrackRecipe, get_date_format
-from calibre.ebooks.BeautifulSoup import BeautifulSoup
from calibre.web.feeds.news import BasicNewsRecipe
_name = "The Diplomat"
@@ -88,7 +87,7 @@ def preprocess_raw_html(self, raw_html, url):
# formulate the api response into html
post = json.loads(raw_html)
post_date = self.parse_date(post["date"], tz_info=None, as_utc=False)
- soup = BeautifulSoup(
+ soup = self.soup(
f"""
@@ -106,25 +105,25 @@ def preprocess_raw_html(self, raw_html, url):
title.string = unescape(post["title"]["rendered"])
soup.body.h1.string = unescape(post["title"]["rendered"])
soup.find("div", class_="sub-headline").append(
- BeautifulSoup(post["excerpt"]["rendered"])
+ self.soup(post["excerpt"]["rendered"])
)
# inject authors
post_authors = self.extract_authors(post)
if post_authors:
soup.find(class_="article-meta").insert(
0,
- BeautifulSoup(f'{", ".join(post_authors)}'),
+ self.soup(f'{", ".join(post_authors)}'),
)
# inject categories
categories = self.extract_categories(post)
if categories:
soup.body.article.insert(
0,
- BeautifulSoup(
+ self.soup(
f'{" / ".join(categories)}'
),
)
- soup.body.article.append(BeautifulSoup(self._extract_featured_media(post)))
+ soup.body.article.append(self.soup(self._extract_featured_media(post)))
return str(soup)
def populate_article_metadata(self, article, soup, first):
diff --git a/recipes/thirdpole.recipe.py b/recipes/thirdpole.recipe.py
index c1e2e030..06246327 100644
--- a/recipes/thirdpole.recipe.py
+++ b/recipes/thirdpole.recipe.py
@@ -11,7 +11,6 @@
sys.path.append(os.environ["recipes_includes"])
from recipes_shared import WordPressNewsrackRecipe, get_datetime_format
-from calibre.ebooks.BeautifulSoup import BeautifulSoup
from calibre.web.feeds.news import BasicNewsRecipe
_name = "The Third Pole"
@@ -73,7 +72,7 @@ def _extract_featured_media(self, post, soup):
:param post_content: Extracted post content
:return:
"""
- post_soup = BeautifulSoup(post["content"]["rendered"])
+ post_soup = self.soup(post["content"]["rendered"])
for img in post_soup.find_all("img", attrs={"data-src": True}):
img["src"] = img["data-src"]
post_content = str(post_soup)
@@ -111,7 +110,7 @@ def preprocess_raw_html(self, raw_html, url):
post_authors = self.extract_authors(post)
categories = self.extract_categories(post)
- soup = BeautifulSoup(
+ soup = self.soup(
f"""
{post["title"]["rendered"]}
@@ -127,9 +126,7 @@ def preprocess_raw_html(self, raw_html, url):
"""
)
- soup.body.article.append(
- BeautifulSoup(self._extract_featured_media(post, soup))
- )
+ soup.body.article.append(self.soup(self._extract_featured_media(post, soup)))
return str(soup)
def parse_index(self):
diff --git a/recipes/time-magazine.recipe.py b/recipes/time-magazine.recipe.py
index 7e66c583..0c3fbdfd 100644
--- a/recipes/time-magazine.recipe.py
+++ b/recipes/time-magazine.recipe.py
@@ -12,7 +12,6 @@
sys.path.append(os.environ["recipes_includes"])
from recipes_shared import BasicNewsrackRecipe, get_date_format
-from calibre.ebooks.BeautifulSoup import BeautifulSoup
from calibre.ptempfile import PersistentTemporaryDirectory, PersistentTemporaryFile
from calibre.web.feeds.news import BasicNewsRecipe
@@ -58,7 +57,7 @@ def preprocess_raw_html(self, raw_html, url):
if not self.pub_date or date_published_utc > self.pub_date:
self.pub_date = date_published_utc
- content_soup = BeautifulSoup(article["content"])
+ content_soup = self.soup(article["content"])
cover_url = self.canonicalize_internal_url(self.cover_url)
# clean up weirdness
div_gmail = content_soup.find_all(name="div", attrs={"class": "gmail_default"})
diff --git a/recipes/wapo-paper.recipe.py b/recipes/wapo-paper.recipe.py
index 6ea6a472..a471086a 100644
--- a/recipes/wapo-paper.recipe.py
+++ b/recipes/wapo-paper.recipe.py
@@ -13,7 +13,6 @@
sys.path.append(os.environ["recipes_includes"])
from recipes_shared import BasicNewsrackRecipe, get_datetime_format
-from calibre.ebooks.BeautifulSoup import BeautifulSoup
from calibre.web.feeds.news import BasicNewsRecipe
from calibre.utils.cleantext import clean_ascii_chars
@@ -73,7 +72,7 @@ def _extract_child_nodes(self, nodes, parent_element, soup, url):
continue
if node_type == "text":
para_ele = soup.new_tag("p")
- para_ele.append(BeautifulSoup(c["content"]))
+ para_ele.append(self.soup(c["content"]))
parent_element.append(para_ele)
elif node_type == "image":
figure_ele = soup.new_tag("figure", attrs={"class": "figure"})
@@ -99,17 +98,17 @@ def _extract_child_nodes(self, nodes, parent_element, soup, url):
parent_element.append(container_ele)
elif node_type == "header":
header_ele = soup.new_tag(f'h{c["level"]}')
- header_ele.append(BeautifulSoup(c["content"]))
+ header_ele.append(self.soup(c["content"]))
parent_element.append(header_ele)
elif node_type == "correction":
para_ele = soup.new_tag("p", attrs={"class": "correction"})
- para_ele.append(BeautifulSoup(c.get("content") or c.get("text")))
+ para_ele.append(self.soup(c.get("content") or c.get("text")))
parent_element.append(para_ele)
elif node_type == "oembed_response":
- embed_ele = BeautifulSoup(c["raw_oembed"]["html"])
+ embed_ele = self.soup(c["raw_oembed"]["html"])
parent_element.append(embed_ele)
elif node_type == "raw_html":
- content = BeautifulSoup(c["content"])
+ content = self.soup(c["content"])
container = content.find("div", attrs={"data-fallback-image-url": True})
if container:
figure_ele = soup.new_tag("figure")
@@ -133,12 +132,12 @@ def _extract_child_nodes(self, nodes, parent_element, soup, url):
) or c.get("header")
if header_string:
header_ele = soup.new_tag("h3")
- header_ele.append(BeautifulSoup(header_string))
+ header_ele.append(self.soup(header_string))
container_ele.append(header_ele)
ol_ele = soup.new_tag("ol")
for i in c.get("items", []):
li_ele = soup.new_tag("li")
- li_ele.append(BeautifulSoup(i["content"]))
+ li_ele.append(self.soup(i["content"]))
ol_ele.append(li_ele)
container_ele.append(ol_ele)
parent_element.append(container_ele)
@@ -151,14 +150,12 @@ def _extract_child_nodes(self, nodes, parent_element, soup, url):
container_ele.append(soup.new_tag("hr", attrs={"class": "story"}))
header_ele = soup.new_tag("h3")
- header_ele.append(
- BeautifulSoup(c.get("headlines", {}).get("basic", ""))
- )
+ header_ele.append(self.soup(c.get("headlines", {}).get("basic", "")))
container_ele.append(header_ele)
# Example 2022-04-13T14:04:03.051Z "%Y-%m-%dT%H:%M:%S.%fZ"
post_date = self.parse_date(c["display_date"])
- meta_ele = BeautifulSoup(
+ meta_ele = self.soup(
f"""
{post_date:{get_datetime_format()}}
@@ -182,7 +179,7 @@ def _extract_child_nodes(self, nodes, parent_element, soup, url):
self.log.debug(json.dumps(c))
def preprocess_raw_html(self, raw_html, url):
- soup = BeautifulSoup(raw_html)
+ soup = self.soup(raw_html)
data = self.get_script_json(soup, "", {"id": "__NEXT_DATA__", "src": False})
content = data.get("props", {}).get("pageProps", {}).get("globalContent", {})
if not content:
@@ -221,7 +218,7 @@ def preprocess_raw_html(self, raw_html, url):
"""
- new_soup = BeautifulSoup(html)
+ new_soup = self.soup(html)
title_ele = new_soup.new_tag("title")
title_ele.append(title)
new_soup.head.append(title_ele)
diff --git a/recipes/wired.recipe.py b/recipes/wired.recipe.py
index 6767a61f..b42deb30 100644
--- a/recipes/wired.recipe.py
+++ b/recipes/wired.recipe.py
@@ -14,7 +14,6 @@
sys.path.append(os.environ["recipes_includes"])
from recipes_shared import BasicCookielessNewsrackRecipe, format_title
-from calibre.ebooks.BeautifulSoup import BeautifulSoup
from calibre.web.feeds.news import BasicNewsRecipe, classes
_name = "Wired Magazine"
@@ -73,7 +72,7 @@ class WiredMagazine(BasicCookielessNewsrackRecipe, BasicNewsRecipe):
]
def preprocess_raw_html(self, raw_html, url):
- soup = BeautifulSoup(raw_html)
+ soup = self.soup(raw_html)
pub_date_meta = soup.find(
name="meta", attrs={"property": "article:published_time"}
)
diff --git a/recipes/world-today.recipe.py b/recipes/world-today.recipe.py
index d1055b5b..8fdd1e90 100644
--- a/recipes/world-today.recipe.py
+++ b/recipes/world-today.recipe.py
@@ -10,7 +10,6 @@
sys.path.append(os.environ["recipes_includes"])
from recipes_shared import BasicNewsrackRecipe
-from calibre.ebooks.BeautifulSoup import BeautifulSoup
from calibre.web.feeds.news import BasicNewsRecipe
_issue_url = ""
@@ -72,7 +71,7 @@ class WorldToday(BasicNewsrackRecipe, BasicNewsRecipe):
"""
def preprocess_raw_html(self, raw_html, url):
- soup = BeautifulSoup(raw_html)
+ soup = self.soup(raw_html)
# find pub date
mod_date_ele = soup.find("meta", attrs={"property": "article:modified_time"})
# Example: 2022-09-30T12:40:17+0100 "%Y-%m-%dT%H:%M:%S%z"
diff --git a/recipes/wsj-paper.recipe.py b/recipes/wsj-paper.recipe.py
index 7b0ba6aa..252ed99d 100644
--- a/recipes/wsj-paper.recipe.py
+++ b/recipes/wsj-paper.recipe.py
@@ -16,7 +16,6 @@
sys.path.append(os.environ["recipes_includes"])
from recipes_shared import BasicNewsrackRecipe, format_title, get_date_format
-from calibre.ebooks.BeautifulSoup import BeautifulSoup
from calibre.web.feeds.news import BasicNewsRecipe, classes
_name = "Wall Street Journal (Print)"
@@ -60,7 +59,7 @@ class WSJ(BasicNewsrackRecipe, BasicNewsRecipe):
]
def preprocess_raw_html(self, raw_html, url):
- soup = BeautifulSoup(raw_html)
+ soup = self.soup(raw_html)
# find pub date
mod_date_ele = soup.find(
"meta", attrs={"name": "article.updated"}