From ce5610a2453d45992a8fff22900c78c513e8a6a0 Mon Sep 17 00:00:00 2001 From: Otto Moerbeek Date: Mon, 17 Feb 2025 12:46:47 +0100 Subject: [PATCH 1/9] rec: generate html-docs using meson compile html-docs --- docs/generate-docs.py | 105 ++++++++++++++++++++++++ pdns/recursordist/docs/generate-docs.py | 1 + pdns/recursordist/meson.build | 13 +++ 3 files changed, 119 insertions(+) create mode 100755 docs/generate-docs.py create mode 120000 pdns/recursordist/docs/generate-docs.py diff --git a/docs/generate-docs.py b/docs/generate-docs.py new file mode 100755 index 000000000000..f820aabee045 --- /dev/null +++ b/docs/generate-docs.py @@ -0,0 +1,105 @@ +"""Generate docs using sphinx in a venv.""" + +import argparse +import glob +import itertools +import subprocess +import sys +import venv +from pathlib import Path + + +def main(): + """Start the script.""" + args = create_argument_parser() + + source_root = args.source_root + build_root = args.build_root + + # Create the venv. + venv_directory = build_root.joinpath(args.venv_name) + venv.create( + venv_directory, + with_pip=True, + clear=True, + upgrade_deps=True, + prompt=args.venv_name, + ) + + # Install some stuff into the venv. + requirements_file = source_root.joinpath(args.requirements_file) + pip = venv_directory.joinpath("bin").joinpath("pip") + subprocess.run([pip, "install", "-U", "pip", "setuptools", "wheel"], check=True) + subprocess.run([pip, "install", "-r", requirements_file], check=True) + + # Run sphinx to generate the man-pages. + source_directory = source_root.joinpath(args.source_directory) + target_directory = build_root.joinpath(args.target_directory) + files = [glob.glob(str(source_root.joinpath(pat))) for pat in args.files] + files = list(itertools.chain.from_iterable(files)) + sphinx_build = venv_directory.joinpath("bin").joinpath("sphinx-build") + subprocess.run( + [ + sphinx_build, + "-b", + "html", + source_directory, + target_directory, + ] + + files, # default is to do all + check=True + ) + + +def create_argument_parser(): + """Create command-line argument parser.""" + parser = argparse.ArgumentParser( + description="Create a virtualenv from a requirements file" + ) + parser.add_argument( + "--build-root", + type=Path, + required=True, + help="Build root", + ) + parser.add_argument( + "--source-root", + type=Path, + required=True, + help="Source root", + ) + parser.add_argument( + "--venv-name", + type=str, + required=True, + help="Name for the virtualenv", + ) + parser.add_argument( + "--requirements-file", + type=Path, + required=True, + help="Package requirements file relative to the source root", + ) + parser.add_argument( + "--source-directory", + type=Path, + required=True, + help="Docs directory relative to the source root (contains conf.py)", + ) + parser.add_argument( + "--target-directory", + type=Path, + required=True, + help="Target directory for man-pages relative to the build root", + ) + parser.add_argument( + "files", + type=Path, + nargs="*", + help="Input files relative to the source root", + ) + return parser.parse_args() + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/pdns/recursordist/docs/generate-docs.py b/pdns/recursordist/docs/generate-docs.py new file mode 120000 index 000000000000..4862bfd5bf11 --- /dev/null +++ b/pdns/recursordist/docs/generate-docs.py @@ -0,0 +1 @@ +../../../docs/generate-docs.py \ No newline at end of file diff --git a/pdns/recursordist/meson.build b/pdns/recursordist/meson.build index e369367bd44f..38ce7c09a7a7 100644 --- a/pdns/recursordist/meson.build +++ b/pdns/recursordist/meson.build @@ -734,3 +734,16 @@ dep_conf_distfile = custom_target( install_dir: get_option('sysconfdir'), ) +run_target( + 'html-docs', + command: [ + python, + product_source_dir / docs_dir / 'generate-docs.py', + '--build-root', '@BUILD_ROOT@', + '--source-root', '@SOURCE_ROOT@', + '--venv-name', 'venv-docs', + '--requirements-file', docs_dir / 'requirements.txt', + '--source-directory', docs_dir, + '--target-directory', '@BUILD_ROOT@' / 'html-docs', + ] +) \ No newline at end of file From 707a2c7c9711d516715f2670bfe606b1a3500c42 Mon Sep 17 00:00:00 2001 From: Otto Moerbeek Date: Mon, 17 Feb 2025 14:59:31 +0100 Subject: [PATCH 2/9] Generate tarball and pdf and introduce all-docs target --- docs/generate-docs.py | 28 ++++++++++++++++++++++----- pdns/recursordist/meson.build | 36 ++++++++++++++++++++++++++++++++--- 2 files changed, 56 insertions(+), 8 deletions(-) diff --git a/docs/generate-docs.py b/docs/generate-docs.py index f820aabee045..871dd19b866d 100755 --- a/docs/generate-docs.py +++ b/docs/generate-docs.py @@ -3,6 +3,7 @@ import argparse import glob import itertools +import os import subprocess import sys import venv @@ -32,24 +33,35 @@ def main(): subprocess.run([pip, "install", "-U", "pip", "setuptools", "wheel"], check=True) subprocess.run([pip, "install", "-r", requirements_file], check=True) - # Run sphinx to generate the man-pages. + # Run sphinx to generate the docs source_directory = source_root.joinpath(args.source_directory) target_directory = build_root.joinpath(args.target_directory) files = [glob.glob(str(source_root.joinpath(pat))) for pat in args.files] files = list(itertools.chain.from_iterable(files)) sphinx_build = venv_directory.joinpath("bin").joinpath("sphinx-build") - subprocess.run( - [ + + if args.latex_name: + buildargs = [ + sphinx_build, + "-M", + "latexpdf", + source_directory, + '.' + ] + else: + buildargs = [ sphinx_build, "-b", "html", source_directory, target_directory, ] - + files, # default is to do all + subprocess.run( + buildargs + files, # default is to do all check=True ) - + if args.latex_name: + os.rename(build_root.joinpath('latex').joinpath(args.latex_name), args.latex_name) def create_argument_parser(): """Create command-line argument parser.""" @@ -92,6 +104,12 @@ def create_argument_parser(): required=True, help="Target directory for man-pages relative to the build root", ) + parser.add_argument( + "--latex-name", + type=Path, + required=False, + help="Generate latex instead of html", + ) parser.add_argument( "files", type=Path, diff --git a/pdns/recursordist/meson.build b/pdns/recursordist/meson.build index 38ce7c09a7a7..1e0b8cb680c5 100644 --- a/pdns/recursordist/meson.build +++ b/pdns/recursordist/meson.build @@ -734,7 +734,7 @@ dep_conf_distfile = custom_target( install_dir: get_option('sysconfdir'), ) -run_target( +html_docs = custom_target( 'html-docs', command: [ python, @@ -745,5 +745,35 @@ run_target( '--requirements-file', docs_dir / 'requirements.txt', '--source-directory', docs_dir, '--target-directory', '@BUILD_ROOT@' / 'html-docs', - ] -) \ No newline at end of file + ], + output: 'html-docs', + console: true, +) + +docs_tarball = custom_target( + 'html-docs.tar.bz2', + command: ['tar', 'cjf', 'html-docs.tar.bz2', html_docs], + output: 'html-docs.tar.bz2', +) + +latex_docs = custom_target( + command: [ + python, + product_source_dir / docs_dir / 'generate-docs.py', + '--build-root', '@BUILD_ROOT@', + '--source-root', '@SOURCE_ROOT@', + '--venv-name', 'venv-docs', + '--requirements-file', docs_dir / 'requirements.txt', + '--source-directory', docs_dir, + '--target-directory', '@BUILD_ROOT@', + '--latex-name', 'PowerDNS-Recursor.pdf', + ], + output: 'PowerDNS-Recursor.pdf', + console: true, +) + +run_target( + 'all-docs', + command: ['echo', 'Generated', html_docs, docs_tarball, latex_docs], +) + From 5f58af5df41808a4bbb10c003f4b35037e1d07d8 Mon Sep 17 00:00:00 2001 From: Otto Moerbeek Date: Mon, 17 Feb 2025 15:29:31 +0100 Subject: [PATCH 3/9] dnsdist docs (html-docs, tarball and pdf) --- pdns/dnsdistdist/docs/generate-docs.py | 1 + pdns/dnsdistdist/meson.build | 45 +++++++++++++++++++++++++- 2 files changed, 45 insertions(+), 1 deletion(-) create mode 120000 pdns/dnsdistdist/docs/generate-docs.py diff --git a/pdns/dnsdistdist/docs/generate-docs.py b/pdns/dnsdistdist/docs/generate-docs.py new file mode 120000 index 000000000000..4862bfd5bf11 --- /dev/null +++ b/pdns/dnsdistdist/docs/generate-docs.py @@ -0,0 +1 @@ +../../../docs/generate-docs.py \ No newline at end of file diff --git a/pdns/dnsdistdist/meson.build b/pdns/dnsdistdist/meson.build index 6900e9bfab6b..63b651bf7037 100644 --- a/pdns/dnsdistdist/meson.build +++ b/pdns/dnsdistdist/meson.build @@ -688,4 +688,47 @@ install_data( 'dnsdist.conf-dist', install_dir : get_option('sysconfdir'), follow_symlinks: true -) \ No newline at end of file +) + +html_docs = custom_target( + 'html-docs', + command: [ + python, + product_source_dir / docs_dir / 'generate-docs.py', + '--build-root', '@BUILD_ROOT@', + '--source-root', '@SOURCE_ROOT@', + '--venv-name', 'venv-docs', + '--requirements-file', docs_dir / 'requirements.txt', + '--source-directory', docs_dir, + '--target-directory', '@BUILD_ROOT@' / 'html-docs', + ], + output: 'html-docs', + console: true, +) + +docs_tarball = custom_target( + 'html-docs.tar.bz2', + command: ['tar', 'cjf', 'html-docs.tar.bz2', html_docs], + output: 'html-docs.tar.bz2', +) + +latex_docs = custom_target( + command: [ + python, + product_source_dir / docs_dir / 'generate-docs.py', + '--build-root', '@BUILD_ROOT@', + '--source-root', '@SOURCE_ROOT@', + '--venv-name', 'venv-docs', + '--requirements-file', docs_dir / 'requirements.txt', + '--source-directory', docs_dir, + '--target-directory', '@BUILD_ROOT@', + '--latex-name', 'dnsdist.pdf', + ], + output: 'dnsdist.pdf', + console: true, +) + +run_target( + 'all-docs', + command: ['echo', html_docs, docs_tarball, latex_docs], +) From 379b8d71dd8dfe04b4c6177bfe4296e8946398d4 Mon Sep 17 00:00:00 2001 From: Otto Moerbeek Date: Mon, 17 Feb 2025 15:47:46 +0100 Subject: [PATCH 4/9] And also do html-docs, tarball and pdf for auth --- docs/generate-docs.py | 10 ++++---- meson.build | 45 +++++++++++++++++++++++++++++++++++ pdns/dnsdistdist/meson.build | 7 +++--- pdns/recursordist/meson.build | 7 +++--- 4 files changed, 58 insertions(+), 11 deletions(-) diff --git a/docs/generate-docs.py b/docs/generate-docs.py index 871dd19b866d..839e4c7f443b 100755 --- a/docs/generate-docs.py +++ b/docs/generate-docs.py @@ -40,7 +40,7 @@ def main(): files = list(itertools.chain.from_iterable(files)) sphinx_build = venv_directory.joinpath("bin").joinpath("sphinx-build") - if args.latex_name: + if args.pdf_name: buildargs = [ sphinx_build, "-M", @@ -60,8 +60,8 @@ def main(): buildargs + files, # default is to do all check=True ) - if args.latex_name: - os.rename(build_root.joinpath('latex').joinpath(args.latex_name), args.latex_name) + if args.pdf_name: + os.rename(build_root.joinpath('latex').joinpath(args.pdf_name), args.pdf_name) def create_argument_parser(): """Create command-line argument parser.""" @@ -105,10 +105,10 @@ def create_argument_parser(): help="Target directory for man-pages relative to the build root", ) parser.add_argument( - "--latex-name", + "--pdf-name", type=Path, required=False, - help="Generate latex instead of html", + help="Generate pdf instead of html", ) parser.add_argument( "files", diff --git a/meson.build b/meson.build index f914ed454038..bd5ff45c3393 100644 --- a/meson.build +++ b/meson.build @@ -1123,3 +1123,48 @@ if get_option('unit-tests-backends') endif endforeach endif + +html_docs = custom_target( + 'html-docs', + command: [ + python, + product_source_dir / docs_dir / 'generate-docs.py', + '--build-root', '@BUILD_ROOT@', + '--source-root', '@SOURCE_ROOT@', + '--venv-name', 'venv-docs', + '--requirements-file', docs_dir / 'requirements.txt', + '--source-directory', docs_dir, + '--target-directory', '@BUILD_ROOT@' / 'html-docs', + ], + output: 'html-docs', + console: true, +) + +docs_tarball = custom_target( + 'html-docs.tar.bz2', + command: ['tar', 'cjf', 'html-docs.tar.bz2', html_docs], + output: 'html-docs.tar.bz2', +) + +pdf_docs = custom_target( + command: [ + python, + product_source_dir / docs_dir / 'generate-docs.py', + '--build-root', '@BUILD_ROOT@', + '--source-root', '@SOURCE_ROOT@', + '--venv-name', 'venv-docs', + '--requirements-file', docs_dir / 'requirements.txt', + '--source-directory', docs_dir, + '--target-directory', '@BUILD_ROOT@', + '--pdf-name', 'PowerDNS-Authoritative.pdf', + ], + output: 'PowerDNS-Authoritative.pdf', + console: true, +) + +run_target( + 'all-docs', + # args mentioned in command line become auto-dependency + command: ['echo', 'Generated', html_docs, docs_tarball, pdf_docs], +) + diff --git a/pdns/dnsdistdist/meson.build b/pdns/dnsdistdist/meson.build index 63b651bf7037..ddf86963d222 100644 --- a/pdns/dnsdistdist/meson.build +++ b/pdns/dnsdistdist/meson.build @@ -712,7 +712,7 @@ docs_tarball = custom_target( output: 'html-docs.tar.bz2', ) -latex_docs = custom_target( +pdf_docs = custom_target( command: [ python, product_source_dir / docs_dir / 'generate-docs.py', @@ -722,7 +722,7 @@ latex_docs = custom_target( '--requirements-file', docs_dir / 'requirements.txt', '--source-directory', docs_dir, '--target-directory', '@BUILD_ROOT@', - '--latex-name', 'dnsdist.pdf', + '--pdf-name', 'dnsdist.pdf', ], output: 'dnsdist.pdf', console: true, @@ -730,5 +730,6 @@ latex_docs = custom_target( run_target( 'all-docs', - command: ['echo', html_docs, docs_tarball, latex_docs], + # args mentioned in command line become auto-dependency + command: ['echo', 'Generated', html_docs, docs_tarball, pdf_docs], ) diff --git a/pdns/recursordist/meson.build b/pdns/recursordist/meson.build index 1e0b8cb680c5..7ddded830670 100644 --- a/pdns/recursordist/meson.build +++ b/pdns/recursordist/meson.build @@ -756,7 +756,7 @@ docs_tarball = custom_target( output: 'html-docs.tar.bz2', ) -latex_docs = custom_target( +pdf_docs = custom_target( command: [ python, product_source_dir / docs_dir / 'generate-docs.py', @@ -766,7 +766,7 @@ latex_docs = custom_target( '--requirements-file', docs_dir / 'requirements.txt', '--source-directory', docs_dir, '--target-directory', '@BUILD_ROOT@', - '--latex-name', 'PowerDNS-Recursor.pdf', + '--pdf-name', 'PowerDNS-Recursor.pdf', ], output: 'PowerDNS-Recursor.pdf', console: true, @@ -774,6 +774,7 @@ latex_docs = custom_target( run_target( 'all-docs', - command: ['echo', 'Generated', html_docs, docs_tarball, latex_docs], + # args mentioned in command line become auto-dependency + command: ['echo', 'Generated', html_docs, docs_tarball, pdf_docs], ) From 112d834a16130c97dc2e40b9290524489e6e3d29 Mon Sep 17 00:00:00 2001 From: Otto Moerbeek Date: Mon, 17 Feb 2025 16:06:42 +0100 Subject: [PATCH 5/9] Only run python if found --- docs/generate-docs.py | 6 +-- meson.build | 83 ++++++++++++++++++----------------- pdns/dnsdistdist/meson.build | 82 +++++++++++++++++----------------- pdns/recursordist/meson.build | 83 ++++++++++++++++++----------------- 4 files changed, 129 insertions(+), 125 deletions(-) diff --git a/docs/generate-docs.py b/docs/generate-docs.py index 839e4c7f443b..6ba273fca303 100755 --- a/docs/generate-docs.py +++ b/docs/generate-docs.py @@ -41,7 +41,7 @@ def main(): sphinx_build = venv_directory.joinpath("bin").joinpath("sphinx-build") if args.pdf_name: - buildargs = [ + build_args = [ sphinx_build, "-M", "latexpdf", @@ -49,7 +49,7 @@ def main(): '.' ] else: - buildargs = [ + build_args = [ sphinx_build, "-b", "html", @@ -57,7 +57,7 @@ def main(): target_directory, ] subprocess.run( - buildargs + files, # default is to do all + build_args + files, # if files is empty, it means do all files check=True ) if args.pdf_name: diff --git a/meson.build b/meson.build index bd5ff45c3393..65d453c14649 100644 --- a/meson.build +++ b/meson.build @@ -1124,47 +1124,48 @@ if get_option('unit-tests-backends') endforeach endif -html_docs = custom_target( - 'html-docs', - command: [ - python, - product_source_dir / docs_dir / 'generate-docs.py', - '--build-root', '@BUILD_ROOT@', - '--source-root', '@SOURCE_ROOT@', - '--venv-name', 'venv-docs', - '--requirements-file', docs_dir / 'requirements.txt', - '--source-directory', docs_dir, - '--target-directory', '@BUILD_ROOT@' / 'html-docs', - ], - output: 'html-docs', - console: true, -) - -docs_tarball = custom_target( - 'html-docs.tar.bz2', - command: ['tar', 'cjf', 'html-docs.tar.bz2', html_docs], - output: 'html-docs.tar.bz2', -) +if python.found() + html_docs = custom_target( + 'html-docs', + command: [ + python, + product_source_dir / docs_dir / 'generate-docs.py', + '--build-root', '@BUILD_ROOT@', + '--source-root', '@SOURCE_ROOT@', + '--venv-name', 'venv-docs', + '--requirements-file', docs_dir / 'requirements.txt', + '--source-directory', docs_dir, + '--target-directory', '@BUILD_ROOT@' / 'html-docs', + ], + output: 'html-docs', + console: true, + ) -pdf_docs = custom_target( - command: [ - python, - product_source_dir / docs_dir / 'generate-docs.py', - '--build-root', '@BUILD_ROOT@', - '--source-root', '@SOURCE_ROOT@', - '--venv-name', 'venv-docs', - '--requirements-file', docs_dir / 'requirements.txt', - '--source-directory', docs_dir, - '--target-directory', '@BUILD_ROOT@', - '--pdf-name', 'PowerDNS-Authoritative.pdf', - ], - output: 'PowerDNS-Authoritative.pdf', - console: true, -) + docs_tarball = custom_target( + 'html-docs.tar.bz2', + command: ['tar', 'cjf', 'html-docs.tar.bz2', html_docs], + output: 'html-docs.tar.bz2', + ) -run_target( - 'all-docs', - # args mentioned in command line become auto-dependency - command: ['echo', 'Generated', html_docs, docs_tarball, pdf_docs], -) + pdf_docs = custom_target( + command: [ + python, + product_source_dir / docs_dir / 'generate-docs.py', + '--build-root', '@BUILD_ROOT@', + '--source-root', '@SOURCE_ROOT@', + '--venv-name', 'venv-docs', + '--requirements-file', docs_dir / 'requirements.txt', + '--source-directory', docs_dir, + '--target-directory', '@BUILD_ROOT@', + '--pdf-name', 'PowerDNS-Authoritative.pdf', + ], + output: 'PowerDNS-Authoritative.pdf', + console: true, + ) + run_target( + 'all-docs', + # args mentioned in command line become auto-dependency + command: ['echo', 'Generated', html_docs, docs_tarball, pdf_docs], + ) +endif diff --git a/pdns/dnsdistdist/meson.build b/pdns/dnsdistdist/meson.build index ddf86963d222..414fafded0ef 100644 --- a/pdns/dnsdistdist/meson.build +++ b/pdns/dnsdistdist/meson.build @@ -690,46 +690,48 @@ install_data( follow_symlinks: true ) -html_docs = custom_target( - 'html-docs', - command: [ - python, - product_source_dir / docs_dir / 'generate-docs.py', - '--build-root', '@BUILD_ROOT@', - '--source-root', '@SOURCE_ROOT@', - '--venv-name', 'venv-docs', - '--requirements-file', docs_dir / 'requirements.txt', - '--source-directory', docs_dir, - '--target-directory', '@BUILD_ROOT@' / 'html-docs', - ], - output: 'html-docs', - console: true, -) +if python.found() + html_docs = custom_target( + 'html-docs', + command: [ + python, + product_source_dir / docs_dir / 'generate-docs.py', + '--build-root', '@BUILD_ROOT@', + '--source-root', '@SOURCE_ROOT@', + '--venv-name', 'venv-docs', + '--requirements-file', docs_dir / 'requirements.txt', + '--source-directory', docs_dir, + '--target-directory', '@BUILD_ROOT@' / 'html-docs', + ], + output: 'html-docs', + console: true, + ) -docs_tarball = custom_target( - 'html-docs.tar.bz2', - command: ['tar', 'cjf', 'html-docs.tar.bz2', html_docs], - output: 'html-docs.tar.bz2', -) + docs_tarball = custom_target( + 'html-docs.tar.bz2', + command: ['tar', 'cjf', 'html-docs.tar.bz2', html_docs], + output: 'html-docs.tar.bz2', + ) -pdf_docs = custom_target( - command: [ - python, - product_source_dir / docs_dir / 'generate-docs.py', - '--build-root', '@BUILD_ROOT@', - '--source-root', '@SOURCE_ROOT@', - '--venv-name', 'venv-docs', - '--requirements-file', docs_dir / 'requirements.txt', - '--source-directory', docs_dir, - '--target-directory', '@BUILD_ROOT@', - '--pdf-name', 'dnsdist.pdf', - ], - output: 'dnsdist.pdf', - console: true, -) + pdf_docs = custom_target( + command: [ + python, + product_source_dir / docs_dir / 'generate-docs.py', + '--build-root', '@BUILD_ROOT@', + '--source-root', '@SOURCE_ROOT@', + '--venv-name', 'venv-docs', + '--requirements-file', docs_dir / 'requirements.txt', + '--source-directory', docs_dir, + '--target-directory', '@BUILD_ROOT@', + '--pdf-name', 'dnsdist.pdf', + ], + output: 'dnsdist.pdf', + console: true, + ) -run_target( - 'all-docs', - # args mentioned in command line become auto-dependency - command: ['echo', 'Generated', html_docs, docs_tarball, pdf_docs], -) + run_target( + 'all-docs', + # args mentioned in command line become auto-dependency + command: ['echo', 'Generated', html_docs, docs_tarball, pdf_docs], + ) +endif \ No newline at end of file diff --git a/pdns/recursordist/meson.build b/pdns/recursordist/meson.build index 7ddded830670..43f583e3fe96 100644 --- a/pdns/recursordist/meson.build +++ b/pdns/recursordist/meson.build @@ -734,47 +734,48 @@ dep_conf_distfile = custom_target( install_dir: get_option('sysconfdir'), ) -html_docs = custom_target( - 'html-docs', - command: [ - python, - product_source_dir / docs_dir / 'generate-docs.py', - '--build-root', '@BUILD_ROOT@', - '--source-root', '@SOURCE_ROOT@', - '--venv-name', 'venv-docs', - '--requirements-file', docs_dir / 'requirements.txt', - '--source-directory', docs_dir, - '--target-directory', '@BUILD_ROOT@' / 'html-docs', - ], - output: 'html-docs', - console: true, -) - -docs_tarball = custom_target( - 'html-docs.tar.bz2', - command: ['tar', 'cjf', 'html-docs.tar.bz2', html_docs], - output: 'html-docs.tar.bz2', -) +if python.found() + html_docs = custom_target( + 'html-docs', + command: [ + python, + product_source_dir / docs_dir / 'generate-docs.py', + '--build-root', '@BUILD_ROOT@', + '--source-root', '@SOURCE_ROOT@', + '--venv-name', 'venv-docs', + '--requirements-file', docs_dir / 'requirements.txt', + '--source-directory', docs_dir, + '--target-directory', '@BUILD_ROOT@' / 'html-docs', + ], + output: 'html-docs', + console: true, + ) -pdf_docs = custom_target( - command: [ - python, - product_source_dir / docs_dir / 'generate-docs.py', - '--build-root', '@BUILD_ROOT@', - '--source-root', '@SOURCE_ROOT@', - '--venv-name', 'venv-docs', - '--requirements-file', docs_dir / 'requirements.txt', - '--source-directory', docs_dir, - '--target-directory', '@BUILD_ROOT@', - '--pdf-name', 'PowerDNS-Recursor.pdf', - ], - output: 'PowerDNS-Recursor.pdf', - console: true, -) + docs_tarball = custom_target( + 'html-docs.tar.bz2', + command: ['tar', 'cjf', 'html-docs.tar.bz2', html_docs], + output: 'html-docs.tar.bz2', + ) -run_target( - 'all-docs', - # args mentioned in command line become auto-dependency - command: ['echo', 'Generated', html_docs, docs_tarball, pdf_docs], -) + pdf_docs = custom_target( + command: [ + python, + product_source_dir / docs_dir / 'generate-docs.py', + '--build-root', '@BUILD_ROOT@', + '--source-root', '@SOURCE_ROOT@', + '--venv-name', 'venv-docs', + '--requirements-file', docs_dir / 'requirements.txt', + '--source-directory', docs_dir, + '--target-directory', '@BUILD_ROOT@', + '--pdf-name', 'PowerDNS-Recursor.pdf', + ], + output: 'PowerDNS-Recursor.pdf', + console: true, + ) + run_target( + 'all-docs', + # args mentioned in command line become auto-dependency + command: ['echo', 'Generated', html_docs, docs_tarball, pdf_docs], + ) +endif From 1fb0df55dc397db87e5523e6c48336ce156d5df0 Mon Sep 17 00:00:00 2001 From: Otto Moerbeek Date: Mon, 17 Feb 2025 16:28:43 +0100 Subject: [PATCH 6/9] Add all '*.rst' files in docs as a dependency. The Meson docs are not a fan of this method, but maintaining that list by hand is too much work and error prone. This does mean that adding existing .rst file to the docs will not be picked up by the dependency checking until you re-setup your build dir. --- meson.build | 5 +++++ pdns/dnsdistdist/meson.build | 5 +++++ pdns/recursordist/meson.build | 5 +++++ 3 files changed, 15 insertions(+) diff --git a/meson.build b/meson.build index 65d453c14649..e4d946e8db35 100644 --- a/meson.build +++ b/meson.build @@ -1125,6 +1125,9 @@ if get_option('unit-tests-backends') endif if python.found() + find_rst_files = run_command(['find', docs_dir, '-name', '*.rst']) + rst_files = find_rst_files.stdout().strip().split('\n') + html_docs = custom_target( 'html-docs', command: [ @@ -1139,6 +1142,7 @@ if python.found() ], output: 'html-docs', console: true, + depend_files: rst_files, ) docs_tarball = custom_target( @@ -1161,6 +1165,7 @@ if python.found() ], output: 'PowerDNS-Authoritative.pdf', console: true, + depend_files: rst_files, ) run_target( diff --git a/pdns/dnsdistdist/meson.build b/pdns/dnsdistdist/meson.build index 414fafded0ef..c13ef069d051 100644 --- a/pdns/dnsdistdist/meson.build +++ b/pdns/dnsdistdist/meson.build @@ -691,6 +691,9 @@ install_data( ) if python.found() + find_rst_files = run_command(['find', docs_dir, '-name', '*.rst']) + rst_files = find_rst_files.stdout().strip().split('\n') + html_docs = custom_target( 'html-docs', command: [ @@ -705,6 +708,7 @@ if python.found() ], output: 'html-docs', console: true, + depend_files: rst_files, ) docs_tarball = custom_target( @@ -727,6 +731,7 @@ if python.found() ], output: 'dnsdist.pdf', console: true, + depend_files: rst_files, ) run_target( diff --git a/pdns/recursordist/meson.build b/pdns/recursordist/meson.build index 43f583e3fe96..c6c4e7a6b5cf 100644 --- a/pdns/recursordist/meson.build +++ b/pdns/recursordist/meson.build @@ -735,6 +735,9 @@ dep_conf_distfile = custom_target( ) if python.found() + find_rst_files = run_command(['find', docs_dir, '-name', '*.rst']) + rst_files = find_rst_files.stdout().strip().split('\n') + html_docs = custom_target( 'html-docs', command: [ @@ -749,6 +752,7 @@ if python.found() ], output: 'html-docs', console: true, + depend_files: rst_files, ) docs_tarball = custom_target( @@ -771,6 +775,7 @@ if python.found() ], output: 'PowerDNS-Recursor.pdf', console: true, + depend_files: rst_files, ) run_target( From 7ffce917e77c43efceb89c4435fcdc15a382f198 Mon Sep 17 00:00:00 2001 From: Otto Moerbeek Date: Tue, 18 Feb 2025 14:20:16 +0100 Subject: [PATCH 7/9] Fix dependencies for docs, using a Sphinx extension to generate a dep file, as suggested by @eli-schwart in https://github.com/PowerDNS/pdns/pull/15169#discussion_r1959138532 Actually, the depfile (sphinx.d) is removed by ninja after generating it as shown by a trace. I do not know why, but touching an .rst file has the correct behaviour. So I suspect it is an optimization. --- docs/conf.py | 11 +++-- docs/depfile.py | 69 +++++++++++++++++++++++++++++++ meson.build | 9 ++-- pdns/dnsdistdist/docs/conf.py | 9 +++- pdns/dnsdistdist/docs/depfile.py | 1 + pdns/dnsdistdist/meson.build | 8 ++-- pdns/recursordist/docs/conf.py | 11 +++-- pdns/recursordist/docs/depfile.py | 1 + pdns/recursordist/meson.build | 10 ++--- 9 files changed, 105 insertions(+), 24 deletions(-) create mode 100644 docs/depfile.py create mode 120000 pdns/dnsdistdist/docs/depfile.py create mode 120000 pdns/recursordist/docs/depfile.py diff --git a/docs/conf.py b/docs/conf.py index 2457532b8324..b8cbbc8bbbc2 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -19,8 +19,8 @@ # # import os import glob -# import sys -# sys.path.insert(0, os.path.abspath('.')) +import sys +from pathlib import Path import guzzle_sphinx_theme import datetime @@ -30,13 +30,15 @@ # # needs_sphinx = '1.0' +sys.path.append(str(Path('.').resolve())) + # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. #extensions = [] #extensions = ['redjack.sphinx.lua', 'sphinxcontrib.httpdomain', 'sphinxjsondomain'] extensions = ['sphinxcontrib.openapi', - 'sphinxcontrib.fulltoc', 'changelog'] + 'sphinxcontrib.fulltoc', 'changelog', 'depfile'] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] @@ -237,3 +239,6 @@ # A list of files that should not be packed into the epub file. epub_exclude_files = ['search.html'] + +depfile = 'sphinx.d' +depfile_stamp = 'sphinx.stamp' diff --git a/docs/depfile.py b/docs/depfile.py new file mode 100644 index 000000000000..fda8ba8340d8 --- /dev/null +++ b/docs/depfile.py @@ -0,0 +1,69 @@ +# coding=utf-8 +# +# QEMU depfile generation extension +# +# Copyright (c) 2020 Red Hat, Inc. +# +# This work is licensed under the terms of the GNU GPLv2 or later. +# See the COPYING file in the top-level directory. + +"""depfile is a Sphinx extension that writes a dependency file for + an external build system""" + +import os +import sphinx +import sys +from pathlib import Path + +__version__ = '1.0' + +def get_infiles(env): + for x in env.found_docs: + yield str(env.doc2path(x)) + yield from ((os.path.join(env.srcdir, dep) + for dep in env.dependencies[x])) + for mod in sys.modules.values(): + if hasattr(mod, '__file__'): + if mod.__file__: + yield mod.__file__ + # this is perhaps going to include unused files: + for static_path in env.config.html_static_path + env.config.templates_path: + for path in Path(static_path).rglob('*'): + yield str(path) + + # also include kdoc script + #yield str(env.config.kerneldoc_bin[1]) + + +def write_depfile(app, exception): + if exception: + return + + env = app.env + if not env.config.depfile: + return + + # Using a directory as the output file does not work great because + # its timestamp does not necessarily change when the contents change. + # So create a timestamp file. + if env.config.depfile_stamp: + with open(env.config.depfile_stamp, 'w') as f: + print('depfile.py: Writing ' + env.config.depfile_stamp) + + with open(env.config.depfile, 'w') as f: + print('depfile.py: Writing ' + env.config.depfile) + print((env.config.depfile_stamp or app.outdir) + ": \\", file=f) + print(*get_infiles(env), file=f) + for x in get_infiles(env): + print(x + ":", file=f) + +def setup(app): + app.add_config_value('depfile', None, 'env') + app.add_config_value('depfile_stamp', None, 'env') + app.connect('build-finished', write_depfile) + + return dict( + version = __version__, + parallel_read_safe = True, + parallel_write_safe = True + ) diff --git a/meson.build b/meson.build index e4d946e8db35..89560ba9613f 100644 --- a/meson.build +++ b/meson.build @@ -1125,9 +1125,6 @@ if get_option('unit-tests-backends') endif if python.found() - find_rst_files = run_command(['find', docs_dir, '-name', '*.rst']) - rst_files = find_rst_files.stdout().strip().split('\n') - html_docs = custom_target( 'html-docs', command: [ @@ -1140,9 +1137,9 @@ if python.found() '--source-directory', docs_dir, '--target-directory', '@BUILD_ROOT@' / 'html-docs', ], - output: 'html-docs', + output: 'sphinx.stamp', console: true, - depend_files: rst_files, + depfile: 'sphinx.d', ) docs_tarball = custom_target( @@ -1165,7 +1162,7 @@ if python.found() ], output: 'PowerDNS-Authoritative.pdf', console: true, - depend_files: rst_files, + depfile: 'sphinx.d', ) run_target( diff --git a/pdns/dnsdistdist/docs/conf.py b/pdns/dnsdistdist/docs/conf.py index 9708c85cd3bc..a137e14707d4 100644 --- a/pdns/dnsdistdist/docs/conf.py +++ b/pdns/dnsdistdist/docs/conf.py @@ -20,6 +20,8 @@ # import os # import sys # sys.path.insert(0, os.path.abspath('.')) +import sys +from pathlib import Path import datetime # -- General configuration ------------------------------------------------ @@ -28,11 +30,13 @@ # # needs_sphinx = '1.0' +sys.path.append(str(Path('.').resolve())) + # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = ['redjack.sphinx.lua', 'sphinxcontrib.httpdomain', 'sphinxjsondomain', - 'sphinxcontrib.fulltoc', 'changelog'] + 'sphinxcontrib.fulltoc', 'changelog', 'depfile'] primary_domain = 'lua' # Add any paths that contain templates here, relative to this directory. @@ -200,4 +204,5 @@ # A list of files that should not be packed into the epub file. epub_exclude_files = ['search.html'] - +depfile = 'sphinx.d' +depfile_stamp = 'sphinx.stamp' diff --git a/pdns/dnsdistdist/docs/depfile.py b/pdns/dnsdistdist/docs/depfile.py new file mode 120000 index 000000000000..0decaf352392 --- /dev/null +++ b/pdns/dnsdistdist/docs/depfile.py @@ -0,0 +1 @@ +../../../docs/depfile.py \ No newline at end of file diff --git a/pdns/dnsdistdist/meson.build b/pdns/dnsdistdist/meson.build index c13ef069d051..e06cfc02a8cf 100644 --- a/pdns/dnsdistdist/meson.build +++ b/pdns/dnsdistdist/meson.build @@ -691,8 +691,6 @@ install_data( ) if python.found() - find_rst_files = run_command(['find', docs_dir, '-name', '*.rst']) - rst_files = find_rst_files.stdout().strip().split('\n') html_docs = custom_target( 'html-docs', @@ -706,9 +704,9 @@ if python.found() '--source-directory', docs_dir, '--target-directory', '@BUILD_ROOT@' / 'html-docs', ], - output: 'html-docs', + output: 'sphinx.stamp', console: true, - depend_files: rst_files, + depfile: 'sphinx.d', ) docs_tarball = custom_target( @@ -731,7 +729,7 @@ if python.found() ], output: 'dnsdist.pdf', console: true, - depend_files: rst_files, + depfile: 'sphinx.d', ) run_target( diff --git a/pdns/recursordist/docs/conf.py b/pdns/recursordist/docs/conf.py index f07504db3253..d0188e72f0d4 100644 --- a/pdns/recursordist/docs/conf.py +++ b/pdns/recursordist/docs/conf.py @@ -18,8 +18,8 @@ # documentation root, use os.path.abspath to make it absolute, like shown here. # # import os -# import sys -# sys.path.insert(0, os.path.abspath('.')) +import sys +from pathlib import Path import guzzle_sphinx_theme import datetime @@ -29,13 +29,15 @@ # # needs_sphinx = '1.0' +sys.path.append(str(Path('.').resolve())) + # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. #extensions = [] #extensions = ['redjack.sphinx.lua', 'sphinxcontrib.httpdomain', 'sphinxjsondomain'] extensions = ['redjack.sphinx.lua', 'sphinxcontrib.httpdomain', 'sphinxjsondomain', - 'sphinxcontrib.fulltoc', 'changelog'] + 'sphinxcontrib.fulltoc', 'changelog', 'depfile'] primary_domain = 'lua' # Add any paths that contain templates here, relative to this directory. @@ -208,3 +210,6 @@ # A list of files that should not be packed into the epub file. epub_exclude_files = ['search.html'] + +depfile = 'sphinx.d' +depfile_stamp = 'sphinx.stamp' diff --git a/pdns/recursordist/docs/depfile.py b/pdns/recursordist/docs/depfile.py new file mode 120000 index 000000000000..0decaf352392 --- /dev/null +++ b/pdns/recursordist/docs/depfile.py @@ -0,0 +1 @@ +../../../docs/depfile.py \ No newline at end of file diff --git a/pdns/recursordist/meson.build b/pdns/recursordist/meson.build index c6c4e7a6b5cf..5d8a970d7b96 100644 --- a/pdns/recursordist/meson.build +++ b/pdns/recursordist/meson.build @@ -735,8 +735,6 @@ dep_conf_distfile = custom_target( ) if python.found() - find_rst_files = run_command(['find', docs_dir, '-name', '*.rst']) - rst_files = find_rst_files.stdout().strip().split('\n') html_docs = custom_target( 'html-docs', @@ -750,9 +748,10 @@ if python.found() '--source-directory', docs_dir, '--target-directory', '@BUILD_ROOT@' / 'html-docs', ], - output: 'html-docs', + output: 'sphinx.stamp', console: true, - depend_files: rst_files, + depfile: 'sphinx.d', + depends: [ metricfiles, recrust ], # for generated .rst files ) docs_tarball = custom_target( @@ -775,7 +774,8 @@ if python.found() ], output: 'PowerDNS-Recursor.pdf', console: true, - depend_files: rst_files, + depfile: 'sphinx.d', + depends: [ metricfiles, recrust ], # for generated .rst files ) run_target( From fe1a59495384d0da8b7d39612ae26b5b33bd0606 Mon Sep 17 00:00:00 2001 From: Otto Moerbeek Date: Tue, 18 Feb 2025 15:50:48 +0100 Subject: [PATCH 8/9] Ignore docs/depfile.py for spell checking --- .github/actions/spell-check/excludes.txt | 1 + .github/actions/spell-check/expect.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/actions/spell-check/excludes.txt b/.github/actions/spell-check/excludes.txt index b6bd8fdb0733..f4a045abc805 100644 --- a/.github/actions/spell-check/excludes.txt +++ b/.github/actions/spell-check/excludes.txt @@ -92,6 +92,7 @@ SUMS$ \.yml$ \.zip$ ^codedocs/doxygen\.conf$ +^docs/depfile.py$ ^docs/http-api/tsigkey\.rst$ ^docs/lua-records/reference/index\.rst$ ^modules/remotebackend/example\.rb$ diff --git a/.github/actions/spell-check/expect.txt b/.github/actions/spell-check/expect.txt index 838f4ef91357..2075d15276c2 100644 --- a/.github/actions/spell-check/expect.txt +++ b/.github/actions/spell-check/expect.txt @@ -284,6 +284,7 @@ defpol defttl Dehaine DENIC +depfile descclassname descname Dessel From 6fd135e48518145458da65e6702e4a3785662c13 Mon Sep 17 00:00:00 2001 From: Otto Moerbeek Date: Tue, 18 Feb 2025 16:42:27 +0100 Subject: [PATCH 9/9] Update .github/actions/spell-check/excludes.txt Co-authored-by: Josh Soref <2119212+jsoref@users.noreply.github.com> --- .github/actions/spell-check/excludes.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/spell-check/excludes.txt b/.github/actions/spell-check/excludes.txt index f4a045abc805..9670cbaa59d7 100644 --- a/.github/actions/spell-check/excludes.txt +++ b/.github/actions/spell-check/excludes.txt @@ -92,7 +92,7 @@ SUMS$ \.yml$ \.zip$ ^codedocs/doxygen\.conf$ -^docs/depfile.py$ +^docs/depfile\.py$ ^docs/http-api/tsigkey\.rst$ ^docs/lua-records/reference/index\.rst$ ^modules/remotebackend/example\.rb$