diff --git a/docs/operator/did_meta.md b/docs/operator/did_meta.md index 6586ed32e01..a5eb16a2a87 100644 --- a/docs/operator/did_meta.md +++ b/docs/operator/did_meta.md @@ -99,3 +99,8 @@ Configuration options for Metadata are: # plugins = list_of_plugins,comma_separated plugins = rucio.core.did_meta_plugins.did_column_meta.DidColumnMeta,escape.rucio.did_meta_plugin ``` + + +## API Documentation + +[The API for existing metadata plugins can be found here](pathname:///html/did_meta_plugins/did_column_meta.html) \ No newline at end of file diff --git a/docs/operator/policy_packages.md b/docs/operator/policy_packages.md index 4cc007d9737..b399886a520 100644 --- a/docs/operator/policy_packages.md +++ b/docs/operator/policy_packages.md @@ -28,6 +28,8 @@ VO. In this case, the configuration parameter or environment variable name is suffixed with the VO name (for example, `package-vo1` or `RUCIO_POLICY_PACKAGE_VO1`). +[The API for policy packages can be found here.](pathname:///html/rse_policies/rsemanager.html) + ## Creating a policy package The basic elements of a policy package are the following: diff --git a/tools/build_documentation.sh b/tools/build_documentation.sh index f5fac29d62f..0678a1b5d5b 100755 --- a/tools/build_documentation.sh +++ b/tools/build_documentation.sh @@ -35,7 +35,10 @@ mkdir -p "$SCRIPT_DIR"/../website/static/html/ cp -r "$AUTO_GENERATED"/rest_api_doc_spec.yaml "$SCRIPT_DIR"/../website/static/yaml/ cp -r "$AUTO_GENERATED"/rest_api_doc.html "$SCRIPT_DIR"/../website/static/html/ -cp -r "$AUTO_GENERATED"/site "$SCRIPT_DIR"/../website/static/html/ +for dir in rse_policies did_meta_plugins transfer_protocols transfer_tools client_api; do + cp -r "$AUTO_GENERATED/$dir" "$SCRIPT_DIR/../website/static/html/" +done + cp -r "$AUTO_GENERATED"/bin "$DOCS" diff --git a/tools/run_in_docker/generate_client_api_docs.sh b/tools/run_in_docker/generate_client_api_docs.sh index 3d732041046..fd0e47e7b69 100755 --- a/tools/run_in_docker/generate_client_api_docs.sh +++ b/tools/run_in_docker/generate_client_api_docs.sh @@ -4,11 +4,13 @@ set -e echo "Generating the Client Api Docs..." -pip install --upgrade mkdocs mkdocs-gen-files mkdocstrings-python mkdocs-material +yq e -i '.docs_dir = "/auto_generated/client_api"' mkdocs.yml +yq e -i '.site_name = "Rucio Python Client Documentation"' mkdocs.yml +pip install --upgrade mkdocs mkdocs-gen-files mkdocstrings-python mkdocs-material mkdir -p /auto_generated/client_api python3 generate_client_api_pages.py -mkdocs build --clean --no-directory-urls +mkdocs build --clean --no-directory-urls --config-file mkdocs.yml --site-dir api_site -cp -r site /auto_generated/ \ No newline at end of file +cp -r api_site /auto_generated/ \ No newline at end of file diff --git a/tools/run_in_docker/generate_docs.sh b/tools/run_in_docker/generate_docs.sh index e4010119d3b..a1a9129db92 100755 --- a/tools/run_in_docker/generate_docs.sh +++ b/tools/run_in_docker/generate_docs.sh @@ -3,7 +3,8 @@ set -e SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) - +wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 +chmod +x /usr/local/bin/yq echo "Generating documentation files..." cd /run_in_docker @@ -20,3 +21,4 @@ cp rucio/etc/docker/test/extra/rucio_sqlite.cfg /opt/rucio/etc/rucio.cfg "$SCRIPT_DIR"/generate_rest_api_docs.sh "$SCRIPT_DIR"/generate_bin_help_docs.sh "$SCRIPT_DIR"/generate_client_api_docs.sh +"$SCRIPT_DIR"/generate_policy_docs.sh diff --git a/tools/run_in_docker/generate_policy_docs.sh b/tools/run_in_docker/generate_policy_docs.sh new file mode 100755 index 00000000000..7e5c2acae5c --- /dev/null +++ b/tools/run_in_docker/generate_policy_docs.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +set -e + +function mkdocs_build { + # Generates the file content for the bin help message. + # + # :param $1: name of directory to store new site + # :param $2: str name of site according to mkdoc.yml + echo "Generating $2..." + yq e -i '.docs_dir = "/auto_generated/'$1'"' mkdocs.yml + yq e -i ".site_name = \"Rucio $2 Documentation\"" mkdocs.yml + + mkdocs build --clean --no-directory-urls --site-dir $1 + cp -r $1 /auto_generated/ + +} + +echo "Setting up for mkdoc site generation..." + +pip install --upgrade mkdocs mkdocs-gen-files mkdocstrings-python mkdocs-material +python3 generate_policy_pages.py + +mkdocs_build "rse_policies" "RSE Policy" +mkdocs_build "transfer_protocols" "Transfer Protocol" +mkdocs_build "transfer_tools" "TransferTool" +mkdocs_build "did_meta_plugins" "DID Meta Plugin" diff --git a/tools/run_in_docker/generate_policy_pages.py b/tools/run_in_docker/generate_policy_pages.py new file mode 100755 index 00000000000..0587f44e8f9 --- /dev/null +++ b/tools/run_in_docker/generate_policy_pages.py @@ -0,0 +1,79 @@ +"""Generate the code reference pages and navigation.""" + +import os +from pathlib import Path + +import mkdocs_gen_files + + +def clean_dir(doc_path): + try: + files = os.listdir(doc_path) + for file in files: + try: + os.remove(f"{doc_path}/{file}") + except IsADirectoryError: + pass # TODO Better handling + except FileNotFoundError: + os.mkdir(doc_path) + + +def write_files(py_files, doc_path, import_prefix, dir_source): + for path in py_files: + module_name = path.name.split(".py")[0] + + try: + print(f"{import_prefix}.{module_name}") + print(f"{doc_path}/{module_name}.md") + full_doc_path = Path(f"{doc_path}/{module_name}.md") + __import__(f"{import_prefix}.{module_name}") + with mkdocs_gen_files.open(full_doc_path, "a") as fd: + module_path = path.relative_to(dir_source).with_suffix("") + fd.write(f"::: {import_prefix}.{module_path}") + fd.write("\n\n") + + except ImportError: + print(f"Could not access page for {import_prefix}.{module_name}") + + +# RSE Policies (w/ abstract base) +policy_src = Path("rucio/lib/rucio/rse") +root = policy_src.parent +doc_path = "/auto_generated/rse_policies" +clean_dir(doc_path) + +py_files = [f for f in list(policy_src.glob("*.py")) if "__init__.py" not in f.name] +write_files(py_files, doc_path, "rucio.rse", policy_src) + +# RSE Transfer Protocols +protocol_src = Path("rucio/lib/rucio/rse/protocols") +root = protocol_src.parent +doc_path = "/auto_generated/transfer_protocols" +clean_dir(doc_path) + +py_files = [f for f in list(protocol_src.glob("*.py")) if "__init__.py" not in f.name] +write_files(py_files, doc_path, "rucio.rse.protocols", protocol_src) + +# Transfer Tools +tools_src = Path( + "rucio/lib/rucio/transfertool/" +) # All the transfer tool policies included +root = tools_src.parent +doc_path = "/auto_generated/transfer_tools" +clean_dir(doc_path) + +py_files = [ + Path(f"{tools_src}/{tool}.py") for tool in ["globus", "fts3"] +] # TODO Verify these are the only supported tools +write_files(py_files, doc_path, "rucio.transfertool", tools_src) + +# Meta Plugins +did_meta = Path("rucio/lib/rucio/core/did_meta_plugins/") +root = did_meta.parent +doc_path = "/auto_generated/did_meta_plugins" +clean_dir(doc_path) + +py_files = [ + f for f in list(did_meta.glob("*_meta.py")) if "__init__.py" not in f.name +] # TODO Better docstrings for these files +write_files(py_files, doc_path, "rucio.core.did_meta_plugins", did_meta) diff --git a/website/docusaurus.config.js b/website/docusaurus.config.js index 4e5e2c34cdb..34cf7ab53af 100644 --- a/website/docusaurus.config.js +++ b/website/docusaurus.config.js @@ -54,7 +54,7 @@ module.exports={ }, "items": [ { - "to": "pathname:///html/site/accountclient.html", + "to": "pathname:///html/client_api/accountclient.html", "label": "Python Client API", "position": "left" }, diff --git a/website/sidebars.json b/website/sidebars.json index bf11677d2da..77d7863f840 100644 --- a/website/sidebars.json +++ b/website/sidebars.json @@ -53,7 +53,17 @@ "operator/transfers/transfers-throttler", "operator/transfers/transfers-submitter", "operator/transfers/configure-rucio-globus", - "operator/transfers/configure-rucio-fts3-plugins" + "operator/transfers/configure-rucio-fts3-plugins", + { + "type": "link", + "label": "TransferTools API", + "href": "pathname:///html/transfer_tools/fts3.html" + }, + { + "type": "link", + "label": "Transfer Protocols API", + "href": "pathname:///html/transfer_protocols/posix.html" + } ] }, "operator/did_meta",