Merge branch 'main' into 434-paginate-queries-run

microbiomedata · Jan 23, 2025 · 4b3da6e · 4b3da6e
2 parents a642c9f + a557d5b
commit 4b3da6e
Show file tree

Hide file tree

Showing 26 changed files with 1,814 additions and 93 deletions.
diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
@@ -4,12 +4,12 @@
     │ Summarize the changes you made on this branch. This is typically a more │
     │ detailed restatement of the PR title.                                   │
     │                                                                         │
-    │ Example: "In this branch, I updated the `/studies/{study_id}` endpoint  │
+    │ Example: "On this branch, I updated the `/studies/{study_id}` endpoint  │
     │           so it returns an HTTP 404 response when the specified study   │
     │           does not exist."                                              │
     └─────────────────────────────────────────────────────────────────────────┘-->
 
-In this branch, I...
+On this branch, I...
 
 ### Details
 

diff --git a/.github/workflows/deploy-redirects.yml b/.github/workflows/deploy-redirects.yml
@@ -0,0 +1,93 @@
+# This GitHub Actions workflow builds a website file tree and deploys it to GitHub Pages.
+# Reference: https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions
+name: Deploy redirects to GitHub Pages
+
+on:
+  push: { branches: [ main ] }
+  workflow_dispatch: { }
+
+# Reference: https://docs.github.com/en/actions/using-jobs/using-concurrency
+concurrency:
+  group: github-pages
+  cancel-in-progress: true
+
+jobs:
+  build:
+    name: Build website
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - name: Check out commit
+        uses: actions/checkout@v4
+      - name: Create website file tree
+        run: |
+          # Define helper function.
+          make_redirect() { 
+            # This function prints the HTML markup for a web page that redirects the client.
+            local url="https://docs.microbiomedata.org/runtime/${1}"
+            echo "<html><head><meta http-equiv=\"refresh\" content=\"0; url=${url}\" /></head><body>Redirecting to <a href=\"${url}\">${url}</a>...</body></html>"
+          }          
+          
+          # Create directories.
+          mkdir -p \
+            _build/html \
+            _build/html/explanation \
+            _build/html/howto-guides \
+            _build/html/howto-guides/jobs \
+            _build/html/tutorials \
+            _build/html/nb
+          
+          # Create HTML files containing redirects.
+          #
+          # Note: These HTML files will be accessible at the same URLs at which _original_ Runtime documentation pages
+          #       had been accessible (on the https://microbiomedata.github.io/nmdc-runtime/ website) prior to the
+          #       launch of the new, NMDC-wide documentation website (i.e., https://docs.microbiomedata.org).
+          #
+          #       For documents added to the Runtime repo's `docs/` directory _after_ the launch of the latter website,
+          #       creating redirects for those documents' URLs is unnecessary, since there was nothing at those URLs
+          #       before (presumably, nobody has expects anything other than an "HTTP 404 Not Found" error at them).
+          #
+          cd _build/html
+          make_redirect index.html                                          > index.html
+          make_redirect admin.html                                          > admin.html
+          make_redirect draft.html                                          > draft.html
+          make_redirect contributing-docs.html                              > contributing-docs.html
+          make_redirect explanation/domain-vision-statement.html            > explanation/domain-vision-statement.html
+          make_redirect explanation/identifiers.html                        > explanation/identifiers.html
+          make_redirect explanation/journeys.html                           > explanation/journeys.html
+          make_redirect howto-guides/update-sensors-ops.html                > howto-guides/update-sensors-ops.html
+          make_redirect howto-guides/create-triggers.html                   > howto-guides/create-triggers.html
+          make_redirect howto-guides/improving-search-api.html              > howto-guides/improving-search-api.html
+          make_redirect howto-guides/release-process.html                   > howto-guides/release-process.html
+          make_redirect howto-guides/author-changesheets.html               > howto-guides/author-changesheets.html
+          make_redirect howto-guides/claim-and-run-jobs.html                > howto-guides/claim-and-run-jobs.html
+          make_redirect howto-guides/jobs/gold-translation-etl.html         > howto-guides/jobs/gold-translation-etl.html
+          make_redirect tutorials/json.html                                 > tutorials/json.html
+          make_redirect tutorials/exporters.html                            > tutorials/exporters.html
+          make_redirect tutorials/metadata-in.html                          > tutorials/metadata-in.html
+          make_redirect tutorials/auth.html                                 > tutorials/auth.html
+          make_redirect tutorials/translators.html                          > tutorials/translators.html
+          make_redirect nb/bulk_validation_referential_integrity_check.html > nb/bulk_validation_referential_integrity_check.html
+          make_redirect nb/get_data.html                                    > nb/get_data.html
+          make_redirect nb/queue_and_trigger_data_jobs.html                 > nb/queue_and_trigger_data_jobs.html
+          make_redirect nb/wf_automation.html                               > nb/wf_automation.html
+      - name: Save the result for publishing to GitHub Pages
+        uses: actions/upload-pages-artifact@v3
+        with:
+          path: _build/html
+  deploy:
+    name: Deploy website
+    needs:
+      - build
+    runs-on: ubuntu-latest
+    permissions:
+      pages: write
+      id-token: write
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4
diff --git a/.github/workflows/mkdocs.yml b/.github/workflows/mkdocs.yml
@@ -20,11 +20,26 @@ jobs:
       - run: pip install git+https://${GH_TOKEN}@github.com/squidfunk/mkdocs-material-insiders.git
       - run: pip install mkdocs-mermaid2-plugin
       - run: pip install mkdocs-jupyter
-      - run: mkdocs gh-deploy --force
+      # Build the MkDocs site (but don't deploy it).
+      #
+      # Note: Until January 17, 2025, we would build and deploy the MkDocs website to GitHub Pages.
+      #       As of January 17, 2025, we just _build_ it — we don't deploy it.
+      #
+      #       The reason is that the NMDC has a new, centralized documentation website: https://docs.microbiomedata.org.
+      #       That website includes a copy of the Runtime documentation (it's at the path, `/runtime`). Instead of
+      #       deploying this redundant copy (being built here) to `https://microbiomedata.github.io/nmdc-runtime`,
+      #       we will (via the new `deploy-redirects.yml` workflow) deploy a bunch of HTML files that redirect visitors
+      #       from pages on `https://microbiomedata.github.io/nmdc-runtime` to their counterparts on
+      #       `https://docs.microbiomedata.org/runtime`.
+      #
+      #       This redirection is not implemented in the MkDocs site, _itself_, because those MkDocs site source files
+      #       are used to build the website hosted at `https://docs.microbiomedata.org/runtime`; and, if a page were
+      #       to redirect to itself, the result would be a circular redirect.
+      #
+      #       The reason we still bother to build the MkDocs site here is so we get an error (a failed GHA workflow run)
+      #       when the site is no longer buildable (e.g. due to someone inadvertently introducing an invalid source
+      #       file, or due to one of the dependencies becoming unavailable), which can prompt remedial action.
+      #
+      - run: mkdocs build
 env:
   GH_TOKEN: ${{ secrets.GH_TOKEN_MKDOCS_MATERIAL_INSIDERS }}
-
-
-
-
-
diff --git a/Makefile b/Makefile
@@ -43,6 +43,8 @@ test-dbinit:
 	docker compose --file docker-compose.test.yml \
 		exec mongo /bin/bash -c "/mongorestore-nmdc-testdb.sh"
 
+# Tip: If you append a file path to this "recipe", pytest will run only the tests defined in that file.
+#      For example, append `tests/test_api/test_endpoints.py` to have pytest only run the endpoint tests.
 test-run:
 	docker compose --file docker-compose.test.yml run test
 
@@ -101,7 +103,7 @@ mongorestore-nmdc-db:
 	mkdir -p /tmp/remote-mongodump/nmdc
 	# Optionally, manually update MONGO_REMOTE_DUMP_DIR env var:
 	# ```bash
-	# export MONGO_REMOTE_DUMP_DIR=$(ssh -i ~/.ssh/nersc -q ${NERSC_USERNAME}@dtn01.nersc.gov 'bash -s ' < get_latest_nmdc_prod_dump_dir.sh 2>/dev/null)
+	# export MONGO_REMOTE_DUMP_DIR=$(ssh -i ~/.ssh/nersc -q ${NERSC_USERNAME}@dtn01.nersc.gov 'bash -s ' < util/get_latest_nmdc_prod_dump_dir.sh 2>/dev/null)
 	# ```
 	# Rsync the remote dump directory items of interest:
 	rsync -av --exclude='_*' --exclude='fs\.*' \