diff --git a/.github/workflows/deploy-docs.yaml b/.github/workflows/deploy-docs.yaml index da85e72b5f..0a218350b4 100644 --- a/.github/workflows/deploy-docs.yaml +++ b/.github/workflows/deploy-docs.yaml @@ -23,8 +23,21 @@ jobs: - name: Install dependencies. run: poetry install -E docs - - name: Build and deploy documentation. - # Build the docs and deploy them to GitHub Pages. + - name: Derive files from sources + # Note: First, we replace the "0.0.0" placeholder version number in `pyproject.toml`. + # Reference: https://github.com/mtkennerly/poetry-dynamic-versioning/blob/master/README.md#command-line-mode + run: | + poetry self add "poetry-dynamic-versioning[plugin]" + poetry dynamic-versioning + make squeaky-clean all + + - name: Generate web-based documentation + run: | + mkdir -p docs + touch docs/.nojekyll + make gendoc + + # Deploy the docs to GitHub Pages. # # Note: The `make mkd-gh-deploy` command below uses the `mkd-%` target # defined in `Makefile`. That target's action (i.e. recipe) @@ -34,8 +47,6 @@ jobs: # # Reference: https://www.mkdocs.org/user-guide/deploying-your-docs/ # + - name: Deploy web-based documentation to GitHub Pages run: | - mkdir -p docs - touch docs/.nojekyll - make gendoc make mkd-gh-deploy diff --git a/.github/workflows/test_pages_build.yaml b/.github/workflows/test_pages_build.yaml index 8d32ab47c7..b6cbc1c69b 100644 --- a/.github/workflows/test_pages_build.yaml +++ b/.github/workflows/test_pages_build.yaml @@ -31,10 +31,18 @@ jobs: - name: Install dependencies run: poetry install -E docs - - name: Build documentation + - name: Derive files from sources + # Note: First, we replace the "0.0.0" placeholder version number in `pyproject.toml`. + # Reference: https://github.com/mtkennerly/poetry-dynamic-versioning/blob/master/README.md#command-line-mode run: | - mkdir -p site - touch site/.nojekyll + poetry self add "poetry-dynamic-versioning[plugin]" + poetry dynamic-versioning + make squeaky-clean all + + - name: Generate web-based documentation + run: | + mkdir -p docs + touch docs/.nojekyll make gendoc poetry run mkdocs build -d site diff --git a/Makefile b/Makefile index 7022a234c1..928260ae3e 100644 --- a/Makefile +++ b/Makefile @@ -155,6 +155,11 @@ $(PYMODEL): $(DOCDIR): mkdir -p $@ +# Compile static Markdown files, images, and JavaScript scripts, into a documentation website. +# +# Then, use `refgraph` (part of `refscan`) to generate a pair of graphs (i.e. network diagrams), +# one that depicts inter-collection relationships and one that depicts inter-class relationships. +# gendoc: $(DOCDIR) # added copying of images and renaming of TEMP.md cp $(SRC)/docs/*md $(DOCDIR) ; \ @@ -162,6 +167,11 @@ gendoc: $(DOCDIR) $(RUN) gen-doc -d $(DOCDIR) --template-directory $(SRC)/$(TEMPLATEDIR) --include src/schema/deprecated.yaml $(SOURCE_SCHEMA_PATH) mkdir -p $(DOCDIR)/javascripts $(RUN) cp $(SRC)/scripts/*.js $(DOCDIR)/javascripts/ + # Use `refgraph` (part of `refscan`) to generate interactive diagrams within the compiled documentation file tree. + # One diagram depicts the relationships between collections and the other depicts the relationships between classes. + mkdir -p $(DOCDIR)/visualizations + $(RUN) refgraph --schema nmdc_schema/nmdc_materialized_patterns.yaml --subject collection --graph $(DOCDIR)/visualizations/collection-graph.html + $(RUN) refgraph --schema nmdc_schema/nmdc_materialized_patterns.yaml --subject class --graph $(DOCDIR)/visualizations/class-graph.html testdoc: gendoc serve diff --git a/mkdocs.yml b/mkdocs.yml index c814d395dd..f6c4040471 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -21,6 +21,7 @@ extra_javascript: - https://unpkg.com/tablesort@5.3.0/dist/tablesort.min.js - javascripts/tablesort.js nav: + # Reference: https://www.mkdocs.org/user-guide/configuration/#nav - NMDC Schema: index.md - About: about.md - How to run a collaborative data modeling project: https://linkml.io/linkml/howtos/collaborative-development.html @@ -30,6 +31,7 @@ nav: - NMDC Schema Contributors How-to from Schema Hackathon 2023-12-10: https://docs.google.com/presentation/d/1ZH41QAoESUwAkdHyUxlrmSKS5M-bT0TOulBgX4rBx2A/edit#slide=id.g26390794265_0_693 - NMDC Schema Validation: schema-validation.md - Schema element deprecation guide: schema_element_deprecation_guide.md + - Visualizations: visualizations.md site_url: https://microbiomedata.github.io/nmdc-schema diff --git a/poetry.lock b/poetry.lock index a72699642a..105e6fd613 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. [[package]] name = "annotated-types" @@ -1534,6 +1534,7 @@ optional = false python-versions = ">=3.6" files = [ {file = "mkdocs-redirects-1.2.1.tar.gz", hash = "sha256:9420066d70e2a6bb357adf86e67023dcdca1857f97f07c7fe450f8f1fb42f861"}, + {file = "mkdocs_redirects-1.2.1-py3-none-any.whl", hash = "sha256:497089f9e0219e7389304cffefccdfa1cac5ff9509f2cb706f4c9b221726dffb"}, ] [package.dependencies] @@ -1785,8 +1786,8 @@ files = [ [package.dependencies] numpy = [ - {version = ">=1.26.0", markers = "python_version >= \"3.12\""}, {version = ">=1.22.4", markers = "python_version < \"3.11\""}, + {version = ">=1.26.0", markers = "python_version >= \"3.12\""}, {version = ">=1.23.2", markers = "python_version == \"3.11\""}, ] python-dateutil = ">=2.8.2" @@ -2028,8 +2029,8 @@ files = [ annotated-types = ">=0.4.0" pydantic-core = "2.20.1" typing-extensions = [ - {version = ">=4.12.2", markers = "python_version >= \"3.13\""}, {version = ">=4.6.1", markers = "python_version < \"3.13\""}, + {version = ">=4.12.2", markers = "python_version >= \"3.13\""}, ] [package.extras] @@ -2226,8 +2227,8 @@ files = [ astroid = ">=3.2.2,<=3.3.0-dev0" colorama = {version = ">=0.4.5", markers = "sys_platform == \"win32\""} dill = [ - {version = ">=0.3.7", markers = "python_version >= \"3.12\""}, {version = ">=0.2", markers = "python_version < \"3.11\""}, + {version = ">=0.3.7", markers = "python_version >= \"3.12\""}, {version = ">=0.3.6", markers = "python_version >= \"3.11\" and python_version < \"3.12\""}, ] isort = ">=4.2.5,<5.13.0 || >5.13.0,<6" @@ -2671,6 +2672,23 @@ files = [ attrs = ">=22.2.0" rpds-py = ">=0.7.0" +[[package]] +name = "refscan" +version = "0.1.21" +description = "Command-line program that scans the NMDC MongoDB database for referential integrity violations" +optional = false +python-versions = "<4.0,>=3.9" +files = [ + {file = "refscan-0.1.21-py3-none-any.whl", hash = "sha256:acb211a5621c5ecec8f9b129e3fcc60ec559898535e0068913d20e4f0c7d5be2"}, + {file = "refscan-0.1.21.tar.gz", hash = "sha256:36ade37ba227ad7c45d5668039d219e41cd8530dbc0400ca9d5a52b60a9207e2"}, +] + +[package.dependencies] +linkml-runtime = ">=1.7.5,<2.0.0" +pymongo = ">=4.7.1,<5.0.0" +rich = ">=13.7.1,<14.0.0" +typer = ">=0.12.3,<0.13.0" + [[package]] name = "regex" version = "2024.5.15" @@ -3119,6 +3137,17 @@ pandas = ">=1.3.4" pandasql = ">=0.7.3,<0.8.0" pygsheets = ">=2.0.5,<3.0.0" +[[package]] +name = "shellingham" +version = "1.5.4" +description = "Tool to Detect Surrounding Shell" +optional = false +python-versions = ">=3.7" +files = [ + {file = "shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686"}, + {file = "shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de"}, +] + [[package]] name = "shexjsg" version = "0.8.2" @@ -3421,6 +3450,23 @@ files = [ {file = "trove_classifiers-2024.7.2.tar.gz", hash = "sha256:8328f2ac2ce3fd773cbb37c765a0ed7a83f89dc564c7d452f039b69249d0ac35"}, ] +[[package]] +name = "typer" +version = "0.12.5" +description = "Typer, build great CLIs. Easy to code. Based on Python type hints." +optional = false +python-versions = ">=3.7" +files = [ + {file = "typer-0.12.5-py3-none-any.whl", hash = "sha256:62fe4e471711b147e3365034133904df3e235698399bc4de2b36c8579298d52b"}, + {file = "typer-0.12.5.tar.gz", hash = "sha256:f592f089bedcc8ec1b974125d64851029c3b1af145f04aca64d69410f0c9b722"}, +] + +[package.dependencies] +click = ">=8.0.0" +rich = ">=10.11.0" +shellingham = ">=1.3.0" +typing-extensions = ">=3.7.4.3" + [[package]] name = "types-python-dateutil" version = "2.9.0.20240316" @@ -3669,4 +3715,4 @@ docs = ["mkdocs-material"] [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "f94dee76c652cdd5b82853a7b7ed550c2c08dc803c5e543cfb4c05224e0ee73f" +content-hash = "75bd51f270387b3f9d69f7087804d4861c5b1847a8a708b6ca42a570efd0ed6f" diff --git a/pyproject.toml b/pyproject.toml index b85eb69dd6..fb3b79600f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,6 +63,7 @@ sheets_and_friends = "^0.5.0" # for do_shuttle (and possibly more) sparql-burger = "^1.0.2" sparql-dataframe = "^0.4" vulture = "^2.11" +refscan = "==0.1.21" #oaklib = "^0.5.24" diff --git a/src/docs/visualizations.md b/src/docs/visualizations.md new file mode 100644 index 0000000000..9f92928337 --- /dev/null +++ b/src/docs/visualizations.md @@ -0,0 +1,25 @@ +# Visualizations + +## Inter-collection relationship diagram + + +This [**inter-collection relationship diagram**](visualizations/collection-graph.html) +shows the database **collections** described by the schema, and the **relationships** between those collections. + +Each circle represents a collection. +Each arrow represents all of the fields that documents in one collection—the one at that arrow's tail—can +use to refer to documents in another collection—the one at that arrow's head. +If you click on a circle, the names of the fields will appear on the arrows connected to that circle. + +## Inter-class relationship diagram + + +This [**inter-class relationship diagram**](visualizations/class-graph.html) +shows the **classes** defined within the schema, and the **relationships** between those classes. + +Each circle represents a class. +Each arrow represents all of the slots that instances of that class—the one at that arrow's tail—can +use to refer to instances of another class—the one at that arrow's head. +If you click on a circle, the names of the slots will appear on the arrows connected to that circle.