From 08f5e37b0c749ea351af6bc57a6f0231a005d64c Mon Sep 17 00:00:00 2001 From: Joonas Somero <50655931+joonas-somero@users.noreply.github.com> Date: Mon, 27 Nov 2023 15:01:48 +0200 Subject: [PATCH] Newsletter/mailing list archives (#1904) * Upgrade MkDocs * Install feedparser * Implement a hook for rendering RSS feeds * Ignore pycache * Only fetch archives when built in production * Fix typo in placeholder text * Add newsletter and mailing list archives * Temporarily enable newsletters in preview Integration tests should fail due to the FIXME * Introduce a FIXME to fail tests * Remove FIXME --- .gitignore | 1 + .../assets/snippets/placeholders/archive.md | 3 + csc-overrides/partials/breadcrumbs.html | 3 + docs/apps/by_system.md | 2 +- docs/apps/index.md | 2 +- .../asiakaskoulutus-customer-training.md | 7 ++ docs/support/archives/lumi-users.md | 7 ++ docs/support/archives/mahti-users.md | 7 ++ docs/support/archives/pouta-users.md | 7 ++ docs/support/archives/puhti-users.md | 7 ++ docs/support/archives/rahti-users.md | 7 ++ docs/support/archives/tutkimus-research.md | 7 ++ docs/support/contact.md | 22 ++-- docs/support/glossary.md | 2 +- .../conda/conda-docs-env-freeze.yaml | 4 +- hooks/archives.py | 118 ++++++++++++++++++ mkdocs.yml | 6 + requirements.txt | 4 +- 18 files changed, 203 insertions(+), 13 deletions(-) create mode 100644 csc-overrides/assets/snippets/placeholders/archive.md create mode 100644 docs/support/archives/asiakaskoulutus-customer-training.md create mode 100644 docs/support/archives/lumi-users.md create mode 100644 docs/support/archives/mahti-users.md create mode 100644 docs/support/archives/pouta-users.md create mode 100644 docs/support/archives/puhti-users.md create mode 100644 docs/support/archives/rahti-users.md create mode 100644 docs/support/archives/tutkimus-research.md create mode 100644 hooks/archives.py diff --git a/.gitignore b/.gitignore index d1267b613c..1a91e09c78 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ site/ *.log .DS_Store .idea/ +__pycache__/ diff --git a/csc-overrides/assets/snippets/placeholders/archive.md b/csc-overrides/assets/snippets/placeholders/archive.md new file mode 100644 index 0000000000..5505269cb5 --- /dev/null +++ b/csc-overrides/assets/snippets/placeholders/archive.md @@ -0,0 +1,3 @@ +This page will be generated during deployment by +the `hooks/archives.py` hook. If this message is visible +after deployment, something has gone wrong. \ No newline at end of file diff --git a/csc-overrides/partials/breadcrumbs.html b/csc-overrides/partials/breadcrumbs.html index 4bab440e64..b3b5671437 100644 --- a/csc-overrides/partials/breadcrumbs.html +++ b/csc-overrides/partials/breadcrumbs.html @@ -3,6 +3,9 @@ | replace("/support/wn/", "/support/whats-new/") %} {# Breadcrumbs are defined literally for pages that don't have ancestors: #} {% set breadcrumbs = { + "/support/archives/": [ + { "url": "/support/contact/#archives", "title": "Archives" } + ], "/support/faq/": [ { "url": "/support/faq/", "title": "FAQ" } ], diff --git a/docs/apps/by_system.md b/docs/apps/by_system.md index cbf7723d55..0b469c99fb 100644 --- a/docs/apps/by_system.md +++ b/docs/apps/by_system.md @@ -1,7 +1,7 @@ # AUTOMATICALLY GENERATED PAGE This page will be generated during deployment by -the `scripts/generate_by_system.sh` script. If this message if visible +the `scripts/generate_by_system.sh` script. If this message is visible after deployment, something has gone wrong. These headings are needed by the tests: diff --git a/docs/apps/index.md b/docs/apps/index.md index c5f8ef5af2..ea8c563900 100644 --- a/docs/apps/index.md +++ b/docs/apps/index.md @@ -1,5 +1,5 @@ # AUTOMATICALLY GENERATED PAGE This page will be generated during deployment by -the `scripts/generate_alpha.sh` script. If this message if visible +the `scripts/generate_alpha.sh` script. If this message is visible after deployment, something has gone wrong. diff --git a/docs/support/archives/asiakaskoulutus-customer-training.md b/docs/support/archives/asiakaskoulutus-customer-training.md new file mode 100644 index 0000000000..01740cb57f --- /dev/null +++ b/docs/support/archives/asiakaskoulutus-customer-training.md @@ -0,0 +1,7 @@ +--- +archive_feed: https://newsletter.csc.fi/archive/asiakaskoulutus-customer-training/rss +--- + +# Newsletter + +--8<-- "placeholders/archive.md" diff --git a/docs/support/archives/lumi-users.md b/docs/support/archives/lumi-users.md new file mode 100644 index 0000000000..77ba3d00d4 --- /dev/null +++ b/docs/support/archives/lumi-users.md @@ -0,0 +1,7 @@ +--- +archive_feed: https://postit.csc.fi/sympa/rss/latest_arc/lumi-users?for=365 +--- + +# Mailing list + +--8<-- "placeholders/archive.md" diff --git a/docs/support/archives/mahti-users.md b/docs/support/archives/mahti-users.md new file mode 100644 index 0000000000..a698630785 --- /dev/null +++ b/docs/support/archives/mahti-users.md @@ -0,0 +1,7 @@ +--- +archive_feed: https://postit.csc.fi/sympa/rss/latest_arc/mahti-users?for=365 +--- + +# Mailing list + +--8<-- "placeholders/archive.md" diff --git a/docs/support/archives/pouta-users.md b/docs/support/archives/pouta-users.md new file mode 100644 index 0000000000..b703d76ee6 --- /dev/null +++ b/docs/support/archives/pouta-users.md @@ -0,0 +1,7 @@ +--- +archive_feed: https://postit.csc.fi/sympa/rss/latest_arc/pouta-users?for=365 +--- + +# Mailing list + +--8<-- "placeholders/archive.md" diff --git a/docs/support/archives/puhti-users.md b/docs/support/archives/puhti-users.md new file mode 100644 index 0000000000..d378204477 --- /dev/null +++ b/docs/support/archives/puhti-users.md @@ -0,0 +1,7 @@ +--- +archive_feed: https://postit.csc.fi/sympa/rss/latest_arc/puhti-users?for=365 +--- + +# Mailing list + +--8<-- "placeholders/archive.md" diff --git a/docs/support/archives/rahti-users.md b/docs/support/archives/rahti-users.md new file mode 100644 index 0000000000..3c66baf119 --- /dev/null +++ b/docs/support/archives/rahti-users.md @@ -0,0 +1,7 @@ +--- +archive_feed: https://postit.csc.fi/sympa/rss/latest_arc/rahti-users?for=365 +--- + +# Mailing list + +--8<-- "placeholders/archive.md" diff --git a/docs/support/archives/tutkimus-research.md b/docs/support/archives/tutkimus-research.md new file mode 100644 index 0000000000..98c7af56ae --- /dev/null +++ b/docs/support/archives/tutkimus-research.md @@ -0,0 +1,7 @@ +--- +archive_feed: https://newsletter.csc.fi/archive/tutkimus-research/rss +--- + +# Newsletter + +--8<-- "placeholders/archive.md" diff --git a/docs/support/contact.md b/docs/support/contact.md index d4928a2f9f..3e9e3130ac 100644 --- a/docs/support/contact.md +++ b/docs/support/contact.md @@ -24,13 +24,19 @@ * [13 point list to speed up resolving your issue](./support-howto.md) -## Mailing list archives +## Archives -Having trouble finding an email sent to you by CSC? Browse the mailing list -archives: +Having trouble finding an email or a newsletter sent to you by CSC? Browse the archives! -* [puhti-users](https://postit.csc.fi/sympa/arc/puhti-users/) -* [mahti-users](https://postit.csc.fi/sympa/arc/mahti-users/) -* [lumi-users](https://postit.csc.fi/sympa/arc/lumi-users/) -* [pouta-users](https://postit.csc.fi/sympa/arc/pouta-users/) -* [rahti-users](https://postit.csc.fi/sympa/arc/rahti-users/) +### Newsletters + +* [Tutkimus / Research](archives/tutkimus-research.md) +* [Asiakaskoulutus / Customer training](archives/asiakaskoulutus-customer-training.md) + +### Mailing lists + +* [puhti-users](archives/puhti-users.md) +* [mahti-users](archives/mahti-users.md) +* [lumi-users](archives/lumi-users.md) +* [pouta-users](archives/pouta-users.md) +* [rahti-users](archives/rahti-users.md) diff --git a/docs/support/glossary.md b/docs/support/glossary.md index 414982e4ac..70fba480a8 100644 --- a/docs/support/glossary.md +++ b/docs/support/glossary.md @@ -1,5 +1,5 @@ # AUTOMATICALLY GENERATED PAGE This page will be generated during deployment by -the `scripts/generate_glossary.sh` script. If this message if visible +the `scripts/generate_glossary.sh` script. If this message is visible after deployment, something has gone wrong. diff --git a/docs/support/tutorials/conda/conda-docs-env-freeze.yaml b/docs/support/tutorials/conda/conda-docs-env-freeze.yaml index 3bde751f67..241f7ecd1c 100644 --- a/docs/support/tutorials/conda/conda-docs-env-freeze.yaml +++ b/docs/support/tutorials/conda/conda-docs-env-freeze.yaml @@ -7,6 +7,7 @@ dependencies: - pip: - Babel==2.10.3 - click==8.1.3 + - feedparser==6.0.10 - ghp-import==2.1.0 - gitdb==4.0.9 - GitPython==3.1.37 @@ -15,7 +16,7 @@ dependencies: - Markdown==3.3.7 - MarkupSafe==2.1.1 - mergedeep==1.3.4 - - mkdocs==1.3.1 + - mkdocs==1.4.3 - mkdocs-git-revision-date-localized-plugin==1.1.0 - mkdocs-material==8.4.0 - mkdocs-material-extensions==1.0.3 @@ -29,6 +30,7 @@ dependencies: - pytz==2022.2.1 - PyYAML==6.0 - pyyaml-env-tag==0.1 + - sgmllib3k==1.0.0 - six==1.16.0 - smmap==5.0.0 - watchdog==2.1.9 diff --git a/hooks/archives.py b/hooks/archives.py new file mode 100644 index 0000000000..1c12abb984 --- /dev/null +++ b/hooks/archives.py @@ -0,0 +1,118 @@ +from http import HTTPStatus +from datetime import date +from logging import getLogger + +import feedparser + + +PRODUCTION_ENVIRONMENT = "production" +ENVIRONMENT_KEY = "environment" +URL_KEY = "archive_feed" + + +class ArchiveItem: + def __init__(self, entry): + self.entry = entry + + def __str__(self): + return self.markdown + + @property + def date(self): + return self.to_date_string(self.entry.published_parsed) + + @property + def href(self): + return self.to_https(self.entry.link) + + @property + def markdown(self): + heading = f"### {self.date}" + link = f"[{self.entry.title}]({self.href}){{ target=_blank }}" + return f"{heading}\n{link}\n" + + @staticmethod + def to_date_string(struct_time): + return "{0:%B} {0:%d}, {0:%Y}".format( + date(*[getattr(struct_time, attr) + for attr + in ("tm_year", "tm_mon", "tm_mday")])) + + @staticmethod + def to_https(href): + if not href.startswith("http"): + raise ValueError(f"Protocol must be HTTP(S): '{href}'") + return (href.replace("http:", "https:", 1) + if href.startswith("http:") + else href) + + +class RSSArchive: + def __init__(self, archive_url): + self.archive = feedparser.parse(archive_url, sanitize_html=False) + + @property + def status(self): + return self.archive.get("status", None) + + @property + def title(self): + return self.archive.feed.title + + @property + def entries(self): + return self.archive.entries + + @property + def items(self): + return map(lambda e: str(ArchiveItem(e)), self.entries) + + @property + def markdown(self): + title = self.title + content = "".join(self.items) + return "## {}\n\n{}".format(title, content) + + +class ArchiveHook: + def __init__(self, logger): + self.logger = logger + self.environment = None + + def __log_status(self, feed): + msg = lambda feed, status: f"Archive '{feed.title}': {status.value} {status.phrase}" + if feed.status is not None: + status = HTTPStatus(feed.status) + message = msg(feed, status) + if 100 <= status.value <= 299: + self.logger.info(message) + elif 300 <= status.value <= 399: + self.logger.warning(message) + elif status.value >= 400: + self.logger.error(message) + + def __get_heading(self, markdown): + return markdown.split("\n")[0] + "\n" + + def on_config(self, config): + self.environment = config.extra[ENVIRONMENT_KEY] + + def on_page_markdown(self, markdown, page, config, files): + if self.environment == PRODUCTION_ENVIRONMENT: + try: + archive_url = page.meta[URL_KEY] + except KeyError: + return None + else: + self.logger.info(f"Fetching archive from {archive_url}...") + archive = RSSArchive(archive_url) + self.__log_status(archive) + page.meta["title"] = archive.title + return self.__get_heading(markdown) + archive.markdown + else: + return None + + +hook = ArchiveHook(getLogger("mkdocs")) +on_config = hook.on_config +on_page_markdown = hook.on_page_markdown diff --git a/mkdocs.yml b/mkdocs.yml index 1d56ce52a8..2a15a6870a 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -5,6 +5,7 @@ site_url: "https://docs.csc.fi/" site_description: Instructions and user guides for the CSC supercomputers, cloud services, storage and software, including FAQ and tutorials extra: + environment: !ENV [MKDOCS_ENV, "preview"] announcement_visible: true # Controls the visibility of the announcement bar breadcrumbs_debug: !ENV DEBUG # Renders a debug view for breadcrumbs navigation analytics: @@ -27,6 +28,9 @@ extra: exclude: - ref.md +hooks: + - hooks/archives.py + plugins: - tags: tags_file: apps/by_license.md @@ -38,6 +42,7 @@ plugins: timezone: Europe/Helsinki exclude: - index.md + - support/archives/*.md - support/glossary.md - support/whats-new.md - apps/index.md @@ -66,6 +71,7 @@ plugins: 'whats-new.md': 'support/whats-new.md' # for convenience: 'glossary.md': 'support/glossary.md' + 'archives.md': 'support/contact.md#archives' markdown_extensions: - footnotes diff --git a/requirements.txt b/requirements.txt index 39224ce964..f6aaedb286 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,11 @@ # This is what we actually install -mkdocs==1.3.1 +mkdocs==1.4.3 mkdocs-git-revision-date-localized-plugin==1.1.0 mkdocs-material==8.4.0 mkdocs-material-extensions==1.0.3 mkdocs-redirects==1.2.0 mkdocs-section-index==0.3.4 +feedparser==6.0.10 # These are dependencies of the above Babel==2.10.3 @@ -25,6 +26,7 @@ python-dateutil==2.8.2 pytz==2022.2.1 PyYAML==6.0 pyyaml-env-tag==0.1 +sgmllib3k==1.0.0 six==1.16.0 smmap==5.0.0 watchdog==2.1.9