Auto generate documentation for policies, transfer protocols, transfe…

…r tools, and did_meta_plugins

docs/operator/did_meta.md

@@ -99,3 +99,8 @@ Configuration options for Metadata are:
 # plugins = list_of_plugins,comma_separated
 plugins = rucio.core.did_meta_plugins.did_column_meta.DidColumnMeta,escape.rucio.did_meta_plugin
 ```
+
+
+## API Documentation
+
+[The API for existing metadata plugins can be found here](pathname:///html/did_meta_plugins/did_column_meta.html)

docs/operator/policy_packages.md

@@ -28,6 +28,8 @@ VO. In this case, the configuration parameter or environment variable name
 is suffixed with the VO name (for example, `package-vo1` or
 `RUCIO_POLICY_PACKAGE_VO1`).
 
+[The API for policy packages can be found here.](pathname:///html/rse_policies/rsemanager.html)
+
 ## Creating a policy package
 
 The basic elements of a policy package are the following:

tools/build_documentation.sh

@@ -35,7 +35,10 @@ mkdir -p "$SCRIPT_DIR"/../website/static/html/
 
 cp -r "$AUTO_GENERATED"/rest_api_doc_spec.yaml "$SCRIPT_DIR"/../website/static/yaml/
 cp -r "$AUTO_GENERATED"/rest_api_doc.html "$SCRIPT_DIR"/../website/static/html/
-cp -r "$AUTO_GENERATED"/site "$SCRIPT_DIR"/../website/static/html/
+for dir in rse_policies did_meta_plugins transfer_protocols transfer_tools client_api; do
+    cp -r "$AUTO_GENERATED/$dir" "$SCRIPT_DIR/../website/static/html/"
+done
+
 cp -r "$AUTO_GENERATED"/bin "$DOCS"
 
 

tools/run_in_docker/generate_client_api_docs.sh

@@ -4,11 +4,13 @@ set -e
 
 echo "Generating the Client Api Docs..."
 
-pip install --upgrade mkdocs mkdocs-gen-files mkdocstrings-python mkdocs-material
+yq e -i '.docs_dir = "/auto_generated/client_api"' mkdocs.yml
+yq e -i '.site_name = "Rucio Python Client Documentation"' mkdocs.yml
 
+pip install --upgrade mkdocs mkdocs-gen-files mkdocstrings-python mkdocs-material
 mkdir -p /auto_generated/client_api
 
 python3 generate_client_api_pages.py
-mkdocs build --clean --no-directory-urls
+mkdocs build --clean --no-directory-urls --config-file mkdocs.yml --site-dir api_site
 
-cp -r site /auto_generated/
+cp -r api_site /auto_generated/

tools/run_in_docker/generate_docs.sh

@@ -3,7 +3,8 @@
 set -e
 
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
-
+wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64
+chmod +x /usr/local/bin/yq
 
 echo "Generating documentation files..."
 cd /run_in_docker
@@ -20,3 +21,4 @@ cp rucio/etc/docker/test/extra/rucio_sqlite.cfg /opt/rucio/etc/rucio.cfg
 "$SCRIPT_DIR"/generate_rest_api_docs.sh
 "$SCRIPT_DIR"/generate_bin_help_docs.sh
 "$SCRIPT_DIR"/generate_client_api_docs.sh
+"$SCRIPT_DIR"/generate_policy_docs.sh

tools/run_in_docker/generate_policy_docs.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+set -e
+
+function mkdocs_build {
+    # Generates the file content for the bin help message.
+    #
+    # :param $1: name of directory to store new site
+    # :param $2: str name of site according to mkdoc.yml
+    echo "Generating $2..."
+    yq e -i '.docs_dir = "/auto_generated/'$1'"' mkdocs.yml
+    yq e -i ".site_name = \"Rucio $2 Documentation\"" mkdocs.yml
+
+    mkdocs build --clean --no-directory-urls --site-dir $1
+    cp -r $1 /auto_generated/
+
+}
+
+echo "Setting up for mkdoc site generation..."
+
+pip install --upgrade mkdocs mkdocs-gen-files mkdocstrings-python mkdocs-material
+python3 generate_policy_pages.py
+
+mkdocs_build "rse_policies" "RSE Policy"
+mkdocs_build "transfer_protocols" "Transfer Protocol"
+mkdocs_build "transfer_tools" "TransferTool"
+mkdocs_build "did_meta_plugins" "DID Meta Plugin"

tools/run_in_docker/generate_policy_pages.py

@@ -0,0 +1,79 @@
+"""Generate the code reference pages and navigation."""
+
+import os
+from pathlib import Path
+
+import mkdocs_gen_files
+
+
+def clean_dir(doc_path):
+    try:
+        files = os.listdir(doc_path)
+        for file in files:
+            try:
+                os.remove(f"{doc_path}/{file}")
+            except IsADirectoryError:
+                pass  # TODO Better handling
+    except FileNotFoundError:
+        os.mkdir(doc_path)
+
+
+def write_files(py_files, doc_path, import_prefix, dir_source):
+    for path in py_files:
+        module_name = path.name.split(".py")[0]
+
+        try:
+            print(f"{import_prefix}.{module_name}")
+            print(f"{doc_path}/{module_name}.md")
+            full_doc_path = Path(f"{doc_path}/{module_name}.md")
+            __import__(f"{import_prefix}.{module_name}")
+            with mkdocs_gen_files.open(full_doc_path, "a") as fd:
+                module_path = path.relative_to(dir_source).with_suffix("")
+                fd.write(f"::: {import_prefix}.{module_path}")
+                fd.write("\n\n")
+
+        except ImportError:
+            print(f"Could not access page for {import_prefix}.{module_name}")
+
+
+# RSE Policies (w/ abstract base)
+policy_src = Path("rucio/lib/rucio/rse")
+root = policy_src.parent
+doc_path = "/auto_generated/rse_policies"
+clean_dir(doc_path)
+
+py_files = [f for f in list(policy_src.glob("*.py")) if "__init__.py" not in f.name]
+write_files(py_files, doc_path, "rucio.rse", policy_src)
+
+# RSE Transfer Protocols
+protocol_src = Path("rucio/lib/rucio/rse/protocols")
+root = protocol_src.parent
+doc_path = "/auto_generated/transfer_protocols"
+clean_dir(doc_path)
+
+py_files = [f for f in list(protocol_src.glob("*.py")) if "__init__.py" not in f.name]
+write_files(py_files, doc_path, "rucio.rse.protocols", protocol_src)
+
+# Transfer Tools
+tools_src = Path(
+    "rucio/lib/rucio/transfertool/"
+)  # All the transfer tool policies included
+root = tools_src.parent
+doc_path = "/auto_generated/transfer_tools"
+clean_dir(doc_path)
+
+py_files = [
+    Path(f"{tools_src}/{tool}.py") for tool in ["globus", "fts3"]
+]  # TODO Verify these are the only supported tools
+write_files(py_files, doc_path, "rucio.transfertool", tools_src)
+
+# Meta Plugins
+did_meta = Path("rucio/lib/rucio/core/did_meta_plugins/")
+root = did_meta.parent
+doc_path = "/auto_generated/did_meta_plugins"
+clean_dir(doc_path)
+
+py_files = [
+    f for f in list(did_meta.glob("*_meta.py")) if "__init__.py" not in f.name
+]  # TODO Better docstrings for these files
+write_files(py_files, doc_path, "rucio.core.did_meta_plugins", did_meta)

website/docusaurus.config.js

@@ -54,7 +54,7 @@ module.exports={
       },
       "items": [
         {
-          "to": "pathname:///html/site/accountclient.html",
+          "to": "pathname:///html/client_api/accountclient.html",
           "label": "Python Client API",
           "position": "left"
         },

website/sidebars.json

@@ -53,7 +53,17 @@
                         "operator/transfers/transfers-throttler",
                         "operator/transfers/transfers-submitter",
                         "operator/transfers/configure-rucio-globus",
-                        "operator/transfers/configure-rucio-fts3-plugins"
+                        "operator/transfers/configure-rucio-fts3-plugins", 
+                        { 
+                            "type": "link", 
+                            "label": "TransferTools API",
+                            "href": "pathname:///html/transfer_tools/fts3.html"
+                        }, 
+                        { 
+                            "type": "link", 
+                            "label": "Transfer Protocols API",
+                            "href": "pathname:///html/transfer_protocols/posix.html"
+                        }
                     ]
                 },
                 "operator/did_meta",