Newsletter/mailing list archives (#1904)

* Upgrade MkDocs

* Install feedparser

* Implement a hook for rendering RSS feeds

* Ignore pycache

* Only fetch archives when built in production

* Fix typo in placeholder text

* Add newsletter and mailing list archives

* Temporarily enable newsletters in preview

Integration tests should fail due to the FIXME

* Introduce a FIXME to fail tests

* Remove FIXME

.gitignore

@@ -5,3 +5,4 @@ site/
 *.log
 .DS_Store
 .idea/
+__pycache__/

csc-overrides/assets/snippets/placeholders/archive.md

@@ -0,0 +1,3 @@
+This page will be generated during deployment by
+the `hooks/archives.py` hook. If this message is visible
+after deployment, something has gone wrong.

csc-overrides/partials/breadcrumbs.html

@@ -3,6 +3,9 @@
   | replace("/support/wn/", "/support/whats-new/") %}
   {# Breadcrumbs are defined literally for pages that don't have ancestors: #}
   {% set breadcrumbs = {
+    "/support/archives/": [
+      { "url": "/support/contact/#archives", "title": "Archives" }
+    ],
     "/support/faq/": [
       { "url": "/support/faq/", "title": "FAQ" }
     ],

docs/apps/by_system.md

@@ -1,7 +1,7 @@
 # AUTOMATICALLY GENERATED PAGE
 
 This page will be generated during deployment by
-the `scripts/generate_by_system.sh` script. If this message if visible
+the `scripts/generate_by_system.sh` script. If this message is visible
 after deployment, something has gone wrong.
 
 These headings are needed by the tests:

docs/apps/index.md

@@ -1,5 +1,5 @@
 # AUTOMATICALLY GENERATED PAGE
 
 This page will be generated during deployment by
-the `scripts/generate_alpha.sh` script. If this message if visible
+the `scripts/generate_alpha.sh` script. If this message is visible
 after deployment, something has gone wrong.

docs/support/archives/asiakaskoulutus-customer-training.md

@@ -0,0 +1,7 @@
+---
+archive_feed: https://newsletter.csc.fi/archive/asiakaskoulutus-customer-training/rss
+---
+
+# Newsletter
+
+--8<-- "placeholders/archive.md"

docs/support/archives/lumi-users.md

@@ -0,0 +1,7 @@
+---
+archive_feed: https://postit.csc.fi/sympa/rss/latest_arc/lumi-users?for=365
+---
+
+# Mailing list
+
+--8<-- "placeholders/archive.md"

docs/support/archives/mahti-users.md

@@ -0,0 +1,7 @@
+---
+archive_feed: https://postit.csc.fi/sympa/rss/latest_arc/mahti-users?for=365
+---
+
+# Mailing list
+
+--8<-- "placeholders/archive.md"

docs/support/archives/pouta-users.md

@@ -0,0 +1,7 @@
+---
+archive_feed: https://postit.csc.fi/sympa/rss/latest_arc/pouta-users?for=365
+---
+
+# Mailing list
+
+--8<-- "placeholders/archive.md"

docs/support/archives/puhti-users.md

@@ -0,0 +1,7 @@
+---
+archive_feed: https://postit.csc.fi/sympa/rss/latest_arc/puhti-users?for=365
+---
+
+# Mailing list
+
+--8<-- "placeholders/archive.md"

docs/support/archives/rahti-users.md

@@ -0,0 +1,7 @@
+---
+archive_feed: https://postit.csc.fi/sympa/rss/latest_arc/rahti-users?for=365
+---
+
+# Mailing list
+
+--8<-- "placeholders/archive.md"

docs/support/archives/tutkimus-research.md

@@ -0,0 +1,7 @@
+---
+archive_feed: https://newsletter.csc.fi/archive/tutkimus-research/rss
+---
+
+# Newsletter
+
+--8<-- "placeholders/archive.md"

docs/support/contact.md

@@ -24,13 +24,19 @@
 
 * [13 point list to speed up resolving your issue](./support-howto.md)
 
-## Mailing list archives
+## Archives
 
-Having trouble finding an email sent to you by CSC? Browse the mailing list
-archives:
+Having trouble finding an email or a newsletter sent to you by CSC? Browse the archives!
 
-* [puhti-users](https://postit.csc.fi/sympa/arc/puhti-users/)
-* [mahti-users](https://postit.csc.fi/sympa/arc/mahti-users/)
-* [lumi-users](https://postit.csc.fi/sympa/arc/lumi-users/)
-* [pouta-users](https://postit.csc.fi/sympa/arc/pouta-users/)
-* [rahti-users](https://postit.csc.fi/sympa/arc/rahti-users/)
+### Newsletters
+
+* [Tutkimus / Research](archives/tutkimus-research.md)
+* [Asiakaskoulutus / Customer training](archives/asiakaskoulutus-customer-training.md)
+
+### Mailing lists
+
+* [puhti-users](archives/puhti-users.md)
+* [mahti-users](archives/mahti-users.md)
+* [lumi-users](archives/lumi-users.md)
+* [pouta-users](archives/pouta-users.md)
+* [rahti-users](archives/rahti-users.md)

docs/support/glossary.md

@@ -1,5 +1,5 @@
 # AUTOMATICALLY GENERATED PAGE
 
 This page will be generated during deployment by
-the `scripts/generate_glossary.sh` script. If this message if visible
+the `scripts/generate_glossary.sh` script. If this message is visible
 after deployment, something has gone wrong.

docs/support/tutorials/conda/conda-docs-env-freeze.yaml

@@ -7,6 +7,7 @@ dependencies:
   - pip:
     - Babel==2.10.3
     - click==8.1.3
+    - feedparser==6.0.10
     - ghp-import==2.1.0
     - gitdb==4.0.9
     - GitPython==3.1.37
@@ -15,7 +16,7 @@ dependencies:
     - Markdown==3.3.7
     - MarkupSafe==2.1.1
     - mergedeep==1.3.4
-    - mkdocs==1.3.1
+    - mkdocs==1.4.3
     - mkdocs-git-revision-date-localized-plugin==1.1.0
     - mkdocs-material==8.4.0
     - mkdocs-material-extensions==1.0.3
@@ -29,6 +30,7 @@ dependencies:
     - pytz==2022.2.1
     - PyYAML==6.0
     - pyyaml-env-tag==0.1
+    - sgmllib3k==1.0.0
     - six==1.16.0
     - smmap==5.0.0
     - watchdog==2.1.9

hooks/archives.py

@@ -0,0 +1,118 @@
+from http import HTTPStatus
+from datetime import date
+from logging import getLogger
+
+import feedparser
+
+
+PRODUCTION_ENVIRONMENT = "production"
+ENVIRONMENT_KEY = "environment"
+URL_KEY = "archive_feed"
+
+
+class ArchiveItem:
+    def __init__(self, entry):
+        self.entry = entry
+
+    def __str__(self):
+        return self.markdown
+
+    @property
+    def date(self):
+        return self.to_date_string(self.entry.published_parsed)
+
+    @property
+    def href(self):
+        return self.to_https(self.entry.link)
+
+    @property
+    def markdown(self):
+        heading = f"### {self.date}"
+        link = f"[{self.entry.title}]({self.href}){{ target=_blank }}"
+        return f"{heading}\n{link}\n"
+
+    @staticmethod
+    def to_date_string(struct_time):
+        return "{0:%B} {0:%d}, {0:%Y}".format(
+            date(*[getattr(struct_time, attr)
+                   for attr
+                   in ("tm_year", "tm_mon", "tm_mday")]))
+
+    @staticmethod
+    def to_https(href):
+        if not href.startswith("http"):
+            raise ValueError(f"Protocol must be HTTP(S): '{href}'")
+        return (href.replace("http:", "https:", 1)
+                if href.startswith("http:")
+                else href)
+
+
+class RSSArchive:
+    def __init__(self, archive_url):
+        self.archive = feedparser.parse(archive_url, sanitize_html=False)
+
+    @property
+    def status(self):
+        return self.archive.get("status", None)
+
+    @property
+    def title(self):
+        return self.archive.feed.title
+
+    @property
+    def entries(self):
+        return self.archive.entries
+
+    @property
+    def items(self):
+        return map(lambda e: str(ArchiveItem(e)), self.entries)
+
+    @property
+    def markdown(self):
+        title = self.title
+        content = "".join(self.items)
+        return "## {}\n\n{}".format(title, content)
+
+
+class ArchiveHook:
+    def __init__(self, logger):
+        self.logger = logger
+        self.environment = None
+
+    def __log_status(self, feed):
+        msg = lambda feed, status: f"Archive '{feed.title}': {status.value} {status.phrase}"
+        if feed.status is not None:
+            status = HTTPStatus(feed.status)
+            message = msg(feed, status)
+            if 100 <= status.value <= 299:
+                self.logger.info(message)
+            elif 300 <= status.value <= 399:
+                self.logger.warning(message)
+            elif status.value >= 400:
+                self.logger.error(message)
+
+    def __get_heading(self, markdown):
+        return markdown.split("\n")[0] + "\n"
+
+    def on_config(self, config):
+        self.environment = config.extra[ENVIRONMENT_KEY]
+
+    def on_page_markdown(self, markdown, page, config, files):
+        if self.environment == PRODUCTION_ENVIRONMENT:
+            try:
+                archive_url = page.meta[URL_KEY]
+            except KeyError:
+                return None
+            else:
+                self.logger.info(f"Fetching archive from {archive_url}...")
+                archive = RSSArchive(archive_url)
+                self.__log_status(archive)
+                page.meta["title"] = archive.title
+                return self.__get_heading(markdown) + archive.markdown
+        else:
+            return None
+
+
+hook = ArchiveHook(getLogger("mkdocs"))
+on_config = hook.on_config
+on_page_markdown = hook.on_page_markdown

mkdocs.yml

@@ -5,6 +5,7 @@ site_url: "https://docs.csc.fi/"
 site_description: Instructions and user guides for the CSC supercomputers, cloud services, storage and software, including FAQ and tutorials
 
 extra:
+  environment: !ENV [MKDOCS_ENV, "preview"]
   announcement_visible: true # Controls the visibility of the announcement bar
   breadcrumbs_debug: !ENV DEBUG # Renders a debug view for breadcrumbs navigation
   analytics:
@@ -27,6 +28,9 @@ extra:
       exclude:
         - ref.md
 
+hooks:
+  - hooks/archives.py
+
 plugins:
   - tags:
       tags_file: apps/by_license.md
@@ -38,6 +42,7 @@ plugins:
       timezone: Europe/Helsinki
       exclude:
         - index.md
+        - support/archives/*.md
         - support/glossary.md
         - support/whats-new.md
         - apps/index.md
@@ -66,6 +71,7 @@ plugins:
         'whats-new.md': 'support/whats-new.md'
         # for convenience:
         'glossary.md': 'support/glossary.md'
+        'archives.md': 'support/contact.md#archives'
 
 markdown_extensions:
   - footnotes

requirements.txt

@@ -1,10 +1,11 @@
 # This is what we actually install
-mkdocs==1.3.1
+mkdocs==1.4.3
 mkdocs-git-revision-date-localized-plugin==1.1.0
 mkdocs-material==8.4.0
 mkdocs-material-extensions==1.0.3
 mkdocs-redirects==1.2.0
 mkdocs-section-index==0.3.4
+feedparser==6.0.10
 
 # These are dependencies of the above
 Babel==2.10.3
@@ -25,6 +26,7 @@ python-dateutil==2.8.2
 pytz==2022.2.1
 PyYAML==6.0
 pyyaml-env-tag==0.1
+sgmllib3k==1.0.0
 six==1.16.0
 smmap==5.0.0
 watchdog==2.1.9