diff --git a/.github/workflows/github-actions.yml b/.github/workflows/github-actions.yml index 0b8a114..cc5decb 100644 --- a/.github/workflows/github-actions.yml +++ b/.github/workflows/github-actions.yml @@ -14,7 +14,7 @@ env: jobs: pytest: runs-on: ubuntu-latest - steps: + steps: - name: Checkout uses: actions/checkout@v3 - name: Set up Python ${{ env.PYTHON_VERSION }} @@ -42,11 +42,11 @@ jobs: run: poetry install - name: pip list run: | - poetry run pip list + poetry run pip list - name: Run tests run: | poetry run python -m pytest -sxv - poetry-update-check: + pre-commit-check: runs-on: ubuntu-latest steps: - name: Checkout @@ -68,29 +68,58 @@ jobs: with: path: ~/.local key: poetry-local-${{ runner.os }}-${{ steps.setup_python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-${{ hashFiles('.github/workflows/*.yml') }} - - name: Install Poetry ${{ env.POETRY_VERSION }} - run: | - curl -sSL ${{ env.POETRY_URL }} | python - --version ${{ env.POETRY_VERSION }} - echo "$HOME/.local/bin" >> $GITHUB_PATH - - name: Install Dependencies - run: poetry install - - name: pip list - run: | - poetry run pip list - - name: Run tests - run: | - poetry run python -m pytest -sxv - - name: poetry update - run: | - poetry update - - name: check if changes - run: | - git status --porcelain - if ! output=$(git status --porcelain) || [ -n "$output" ]; then - echo "*** 'poetry update' produced changes to the repository; these changes should be checked in ***"; - git --no-pager diff; - exit 1; - fi + # TODO: just use poetry to install pre-commit? chicken before egg situation... + - name: Install pre-commit + run: pip3 install pre-commit + - name: Run pre-commit + # skip poetry-lock because it breaks (aka changes) when run on different platforms + # - the pre-commit on developer's hosts should catch if it's out of date (the goal) + run: SKIP=poetry-lock pre-commit run -av + # poetry-update-check: + # runs-on: ubuntu-latest + # steps: + # - name: Checkout + # uses: actions/checkout@v3 + # - name: Set up Python ${{ env.PYTHON_VERSION }} + # uses: actions/setup-python@v4 + # with: + # python-version: ${{ env.PYTHON_VERSION }} + # id: setup_python + # # Poetry cache depends on OS, Python version and Poetry version. + # - name: Cache Poetry cache + # uses: actions/cache@v3 + # with: + # path: ~/.cache/pypoetry + # key: poetry-cache-${{ runner.os }}-${{ steps.setup_python.outputs.python-version }}-${{ env.POETRY_VERSION }} + # # virtualenv cache should depends on OS, Python version and `poetry.lock` (and optionally workflow files). + # - name: Cache Packages + # uses: actions/cache@v3 + # with: + # path: ~/.local + # key: poetry-local-${{ runner.os }}-${{ steps.setup_python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-${{ hashFiles('.github/workflows/*.yml') }} + # - name: Install Poetry ${{ env.POETRY_VERSION }} + # run: | + # curl -sSL ${{ env.POETRY_URL }} | python - --version ${{ env.POETRY_VERSION }} + # echo "$HOME/.local/bin" >> $GITHUB_PATH + # - name: Install Dependencies + # run: poetry install + # - name: pip list + # run: | + # poetry run pip list + # - name: Run tests + # run: | + # poetry run python -m pytest -sxv + # - name: poetry update + # run: | + # poetry update + # - name: check if changes + # run: | + # git status --porcelain + # if ! output=$(git status --porcelain) || [ -n "$output" ]; then + # echo "*** 'poetry update' produced changes to the repository; these changes should be checked in ***"; + # git --no-pager diff; + # exit 1; + # fi monopacker-validate: runs-on: ubuntu-latest steps: @@ -100,7 +129,7 @@ jobs: uses: hashicorp/setup-packer@main id: setup with: - version: ${{ env.PACKER_VERSION }} + version: ${{ env.PACKER_VERSION }} - name: Set up Python ${{ env.PYTHON_VERSION }} uses: actions/setup-python@v4 with: @@ -134,4 +163,4 @@ jobs: poetry run monopacker validate "$builder_instance" done echo "" - echo "ALL VALIDATED" \ No newline at end of file + echo "ALL VALIDATED" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..4e0ef8b --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,21 @@ +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.5.0 + hooks: + - id: check-yaml + - id: end-of-file-fixer + - id: trailing-whitespace +- repo: https://github.com/psf/black + rev: 23.11.0 + hooks: + - id: black +- repo: https://github.com/python-poetry/poetry + rev: '8dbe41e' + hooks: + - id: poetry-check + args: ["--lock"] + - id: poetry-lock +- repo: https://github.com/aerickson/poetry + rev: '0113a0c' + hooks: + - id: poetry-update diff --git a/LICENSE b/LICENSE index c33dcc7..82b4de9 100644 --- a/LICENSE +++ b/LICENSE @@ -351,4 +351,3 @@ Exhibit B - “Incompatible With Secondary Licenses” Notice This Source Code Form is “Incompatible With Secondary Licenses”, as defined by the Mozilla Public License, v. 2.0. - diff --git a/README.md b/README.md index 60d7add..e953fdb 100644 --- a/README.md +++ b/README.md @@ -124,4 +124,4 @@ Mostly, I just haven't tried to make this work. To run the tests for this library, run `poetry run pytest`. -To update dependencies, run `poetry update`. \ No newline at end of file +To update dependencies, run `poetry update`. diff --git a/bin/monopacker b/bin/monopacker index 7b9e136..3ea572c 100755 --- a/bin/monopacker +++ b/bin/monopacker @@ -2,4 +2,5 @@ if __name__ == "__main__": from monopacker.main import main + main() diff --git a/monopacker/files.py b/monopacker/files.py index 27f76d9..d4998fb 100644 --- a/monopacker/files.py +++ b/monopacker/files.py @@ -3,7 +3,8 @@ import os, sys, tarfile, tempfile from pathlib import Path + def pack_files(files_dir, files_tar): # create a directory with `files` as the top level directory component. with tarfile.open(files_tar, "w") as tar: - tar.add(files_dir, arcname='files') + tar.add(files_dir, arcname="files") diff --git a/monopacker/main.py b/monopacker/main.py index a6bdc3e..63b2543 100644 --- a/monopacker/main.py +++ b/monopacker/main.py @@ -11,13 +11,15 @@ generate_packer_template_params, ) + @click.group() def main(): """ Manages building worker images for Taskcluster. """ -@main.command(name='packer-template') + +@main.command(name="packer-template") @generate_packer_template_params def packer_template(**kwargs): """This tool expects a jinja2 templated packer.yaml and the name of one or more builders. @@ -27,18 +29,20 @@ def packer_template(**kwargs): packer_template = generate_packer_template(**kwargs) print(json.dumps(packer_template, sort_keys=True, indent=4)) -@main.command(name='validate') + +@main.command(name="validate") @generate_packer_template_params @run_packer_params def validate(**kwargs): """Validate the generated template with 'packer validate'""" packer_template = generate_packer_template(**kwargs) - run_packer('validate', packer_template, **kwargs) + run_packer("validate", packer_template, **kwargs) + -@main.command(name='build') +@main.command(name="build") @generate_packer_template_params @run_packer_params def validate(**kwargs): """Validate the generated template with 'packer validate'""" packer_template = generate_packer_template(**kwargs) - run_packer('build', packer_template, **kwargs) + run_packer("build", packer_template, **kwargs) diff --git a/monopacker/packer.py b/monopacker/packer.py index ada5fab..36b1f04 100644 --- a/monopacker/packer.py +++ b/monopacker/packer.py @@ -5,33 +5,35 @@ import json import shlex + def run_packer_params(fn): "Decorate a click function with options for run_packer" params = [ click.option( - "--packer", - type=str, - help="path to the packer binary", - default='packer'), + "--packer", type=str, help="path to the packer binary", default="packer" + ), click.option( "--packer-args", type=str, help="additional arguments to pass to packer (shell-quoted)", - default='') - ] + default="", + ), + ] params.reverse() for param in params: fn = param(fn) return fn + def run_packer(command, input, *, packer, packer_args, **_): """Run packer with the given (JSON) template, passing additional extra_args""" packer_args = shlex.split(packer_args) try: subprocess.run( - [packer, command] + packer_args + ['-'], - input=bytes(json.dumps(input), 'utf8'), - check=True) + [packer, command] + packer_args + ["-"], + input=bytes(json.dumps(input), "utf8"), + check=True, + ) return True except subprocess.CalledProcessError as e: print(f"packer exited with code {e.returncode}", file=sys.stderr) diff --git a/monopacker/secrets.py b/monopacker/secrets.py index 5584b9d..fad700c 100755 --- a/monopacker/secrets.py +++ b/monopacker/secrets.py @@ -7,6 +7,7 @@ yaml = YAML(typ="safe") + def pack_secrets(secrets_file, secrets_tar): with open(secrets_file, "r") as f: secrets = yaml.load(f) @@ -22,7 +23,7 @@ def pack_secrets(secrets_file, secrets_tar): raise RuntimeError(f"Encountered secret {name} without `path` key") if "value" not in secret: raise RuntimeError(f"Encountered secret {name} without `value` key") - path = secret["path"].lstrip('/') + path = secret["path"].lstrip("/") value = bytes(secret["value"], "utf8") ti = tarfile.TarInfo(path) ti.size = len(value) diff --git a/monopacker/template_packer.py b/monopacker/template_packer.py index 9c8d158..5244c10 100755 --- a/monopacker/template_packer.py +++ b/monopacker/template_packer.py @@ -20,6 +20,7 @@ yaml = YAML(typ="safe") + def get_files_from_subdirs(*args, root_dir=".", globs=["*"]): """Get an sorted list of files from a list of subdirectories @@ -59,8 +60,8 @@ def load_yaml_from_file(filename: str): def merge_vars(base_vars, override_vars): """Takes two dicts, returns a new dict. Values in the second dict take precedence. - If both dicts have a dictionary value for a key their subdicts are - merged, recursively. All other values (including lists) are overridden. + If both dicts have a dictionary value for a key their subdicts are + merged, recursively. All other values (including lists) are overridden. """ d = {**base_vars} for k, v in override_vars.items(): @@ -74,7 +75,7 @@ def merge_vars(base_vars, override_vars): def get_vars_from_files(files: Sequence[str]): """Takes a list of variable files - in `root_dir` of increasing precedence, returns a merged dict + in `root_dir` of increasing precedence, returns a merged dict """ d = {} for file in files: @@ -174,51 +175,56 @@ def get_builders_for_templating( ) return builders + def generate_packer_template_params(fn): "Decorate a click function with options for generate_packer_template" params = [ - click.argument( - 'builders', - nargs=-1, - type=str, - required=True), + click.argument("builders", nargs=-1, type=str, required=True), click.option( "--builders_dir", type=str, help="directory for builder configuration", - default=os.environ.get("MONOPACKER_BUILDERS_DIR", "./builders")), + default=os.environ.get("MONOPACKER_BUILDERS_DIR", "./builders"), + ), click.option( "--var_files_dir", type=str, help="directory for builder var_files", - default=os.environ.get("MONOPACKER_VARS_DIR", "./template/vars")), + default=os.environ.get("MONOPACKER_VARS_DIR", "./template/vars"), + ), click.option( "--templates_dir", type=str, help="directory for builder templates", - default=os.environ.get("MONOPACKER_TEMPLATES_DIR", "./template/builders")), + default=os.environ.get("MONOPACKER_TEMPLATES_DIR", "./template/builders"), + ), click.option( "--scripts_dir", type=str, help="directory for builder templates", - default=os.environ.get("MONOPACKER_SCRIPTS_DIR", "./scripts")), + default=os.environ.get("MONOPACKER_SCRIPTS_DIR", "./scripts"), + ), click.option( "--files_dir", type=str, help="directory for binary files used in packer provisioners", - default=os.environ.get("MONOPACKER_FILES_DIR", "./files")), + default=os.environ.get("MONOPACKER_FILES_DIR", "./files"), + ), click.option( "--secrets_file", type=str, help="file containing secrets", - default='./fake_secrets.yaml'), - ] + default="./fake_secrets.yaml", + ), + ] params.reverse() for param in params: fn = param(fn) return fn -def generate_packer_template(*, + +def generate_packer_template( + *, builders, builders_dir, var_files_dir, @@ -226,9 +232,10 @@ def generate_packer_template(*, scripts_dir, files_dir, secrets_file, - **_): - pack_secrets(secrets_file, 'secrets.tar') - pack_files(files_dir, 'files.tar') + **_, +): + pack_secrets(secrets_file, "secrets.tar") + pack_files(files_dir, "files.tar") # variables namespaced per builder variables: Dict[str, Dict[str, Any]] = {} @@ -261,50 +268,59 @@ def generate_packer_template(*, # include some setup for linux and windows builders if linux_builders: - pkr["provisioners"].append({ - "type": "file", - "source": "./files.tar", - "destination": "/tmp/", - # TODO: only - }) - pkr["provisioners"].append({ - "type": "shell", - "inline": [ - # files.tar is two levels deep (/tmp/files) - "sudo tar xvf /tmp/files.tar -C / --strip-components=1", - "rm /tmp/files.tar", - ], - # TODO: only - }) - pkr["provisioners"].append({ - 'type': 'file', - 'source': './secrets.tar', - 'destination': '/tmp/', - # TODO: only - }) - pkr["provisioners"].append({ - 'type': 'shell', - 'inline': [ - 'sudo mkdir -p /etc/taskcluster/secrets', - 'sudo tar xvf /tmp/secrets.tar -C /', - 'sudo chown root:root -R /etc/taskcluster', - 'sudo chmod 0400 -R /etc/taskcluster/secrets', - 'rm /tmp/secrets.tar', - ], - 'only': linux_builders, - }) - pkr["provisioners"].append({ - 'type': 'shell', - 'inline': [ - '/usr/bin/cloud-init status --wait', - ], - 'only': linux_builders, - }) + pkr["provisioners"].append( + { + "type": "file", + "source": "./files.tar", + "destination": "/tmp/", + # TODO: only + } + ) + pkr["provisioners"].append( + { + "type": "shell", + "inline": [ + # files.tar is two levels deep (/tmp/files) + "sudo tar xvf /tmp/files.tar -C / --strip-components=1", + "rm /tmp/files.tar", + ], + # TODO: only + } + ) + pkr["provisioners"].append( + { + "type": "file", + "source": "./secrets.tar", + "destination": "/tmp/", + # TODO: only + } + ) + pkr["provisioners"].append( + { + "type": "shell", + "inline": [ + "sudo mkdir -p /etc/taskcluster/secrets", + "sudo tar xvf /tmp/secrets.tar -C /", + "sudo chown root:root -R /etc/taskcluster", + "sudo chmod 0400 -R /etc/taskcluster/secrets", + "rm /tmp/secrets.tar", + ], + "only": linux_builders, + } + ) + pkr["provisioners"].append( + { + "type": "shell", + "inline": [ + "/usr/bin/cloud-init status --wait", + ], + "only": linux_builders, + } + ) e = Environment(loader=FileSystemLoader([templates_dir])) e.filters["clean_gcp_image_name"] = clean_gcp_image_name for builder in templated_builders: - # for each monopacker builder, use the Jinja template to generate a Packer builder template_file = Path(templates_dir) / (builder["template"] + ".jinja2") with open(template_file) as f: @@ -338,7 +354,9 @@ def generate_packer_template(*, sys.exit(1) if type(template_builders) != list: - print(f"Template {template_file} generated YAML that is not an array:\n{output}\n") + print( + f"Template {template_file} generated YAML that is not an array:\n{output}\n" + ) print(f"Packer template variables:\n{variables}\n") sys.exit(1) @@ -347,26 +365,40 @@ def generate_packer_template(*, # make a provisioner for each builder, specialized to run only on that builder, # with that builder's scripts and variables if linux_builders: - pkr["provisioners"].append({ - 'type': 'shell', - 'scripts': builder["scripts"], - 'environment_vars': builder["vars"]["env_vars"] if "env_vars" in builder["vars"] else None, - 'execute_command': builder["vars"]["execute_command"] if "execute_command" in builder["vars"] else None, - 'expect_disconnect': True, - 'start_retry_timeout': builder["vars"]["ssh_timeout"] if "ssh_timeout" in builder["vars"] else None, - 'only': [builder["vars"]["name"]] if builder["platform"] == "linux" else [], - }) + pkr["provisioners"].append( + { + "type": "shell", + "scripts": builder["scripts"], + "environment_vars": builder["vars"]["env_vars"] + if "env_vars" in builder["vars"] + else None, + "execute_command": builder["vars"]["execute_command"] + if "execute_command" in builder["vars"] + else None, + "expect_disconnect": True, + "start_retry_timeout": builder["vars"]["ssh_timeout"] + if "ssh_timeout" in builder["vars"] + else None, + "only": [builder["vars"]["name"]] + if builder["platform"] == "linux" + else [], + } + ) if windows_builders: - pkr["provisioners"].append({ - 'type': 'powershell', - 'scripts': builder["scripts"], - 'only': [builder["vars"]["name"]] if builder["platform"] == "windows" else [], - }) + pkr["provisioners"].append( + { + "type": "powershell", + "scripts": builder["scripts"], + "only": [builder["vars"]["name"]] + if builder["platform"] == "windows" + else [], + } + ) # ensure we output the expected artifacts.. - pkr["post-processors"] = [ - {'type': 'manifest', 'output': 'packer-artifacts.json', 'strip_path': True}, + pkr["post-processors"] = [ + {"type": "manifest", "output": "packer-artifacts.json", "strip_path": True}, ] return pkr diff --git a/poetry.lock b/poetry.lock index 6f691ca..6584b20 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.7.0 and should not be changed by hand. [[package]] name = "click" @@ -193,24 +193,24 @@ testing = ["pytest", "pytest-benchmark"] [[package]] name = "pyfakefs" -version = "5.2.4" +version = "5.3.0" description = "pyfakefs implements a fake file system that mocks the Python file system modules." optional = false python-versions = ">=3.7" files = [ - {file = "pyfakefs-5.2.4-py3-none-any.whl", hash = "sha256:8eb95f1dd1c4b8bdce30448fe169875e3a4451c32d3f9c37799157bd4eb7b789"}, - {file = "pyfakefs-5.2.4.tar.gz", hash = "sha256:3e040f3792086086a0dc2191b05fe709438e168aafe2e94fcdbef8e3859208d8"}, + {file = "pyfakefs-5.3.0-py3-none-any.whl", hash = "sha256:33c1f891078c727beec465e75cb314120635e2298456493cc2cc0539e2130cbb"}, + {file = "pyfakefs-5.3.0.tar.gz", hash = "sha256:e3e35f65ce55ee8ecc5e243d55cfdbb5d0aa24938f6e04e19f0fab062f255020"}, ] [[package]] name = "pytest" -version = "7.4.2" +version = "7.4.3" description = "pytest: simple powerful testing with Python" optional = false python-versions = ">=3.7" files = [ - {file = "pytest-7.4.2-py3-none-any.whl", hash = "sha256:1d881c6124e08ff0a1bb75ba3ec0bfd8b5354a01c194ddd5a0a870a48d99b002"}, - {file = "pytest-7.4.2.tar.gz", hash = "sha256:a766259cfab564a2ad52cb1aae1b881a75c3eb7e34ca3779697c23ed47c47069"}, + {file = "pytest-7.4.3-py3-none-any.whl", hash = "sha256:0d009c083ea859a71b76adf7c1d502e4bc170b80a8ef002da5806527b9591fac"}, + {file = "pytest-7.4.3.tar.gz", hash = "sha256:d989d136982de4e3b29dabcc838ad581c64e8ed52c11fbe86ddebd9da0818cd5"}, ] [package.dependencies] @@ -245,48 +245,61 @@ jinja2 = ["ruamel.yaml.jinja2 (>=0.2)"] [[package]] name = "ruamel-yaml-clib" -version = "0.2.7" +version = "0.2.8" description = "C version of reader, parser and emitter for ruamel.yaml derived from libyaml" optional = false -python-versions = ">=3.5" +python-versions = ">=3.6" files = [ - {file = "ruamel.yaml.clib-0.2.7-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d5859983f26d8cd7bb5c287ef452e8aacc86501487634573d260968f753e1d71"}, - {file = "ruamel.yaml.clib-0.2.7-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:debc87a9516b237d0466a711b18b6ebeb17ba9f391eb7f91c649c5c4ec5006c7"}, - {file = "ruamel.yaml.clib-0.2.7-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:df5828871e6648db72d1c19b4bd24819b80a755c4541d3409f0f7acd0f335c80"}, - {file = "ruamel.yaml.clib-0.2.7-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:efa08d63ef03d079dcae1dfe334f6c8847ba8b645d08df286358b1f5293d24ab"}, - {file = "ruamel.yaml.clib-0.2.7-cp310-cp310-win32.whl", hash = "sha256:763d65baa3b952479c4e972669f679fe490eee058d5aa85da483ebae2009d231"}, - {file = "ruamel.yaml.clib-0.2.7-cp310-cp310-win_amd64.whl", hash = "sha256:d000f258cf42fec2b1bbf2863c61d7b8918d31ffee905da62dede869254d3b8a"}, - {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:045e0626baf1c52e5527bd5db361bc83180faaba2ff586e763d3d5982a876a9e"}, - {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-macosx_13_0_arm64.whl", hash = "sha256:1a6391a7cabb7641c32517539ca42cf84b87b667bad38b78d4d42dd23e957c81"}, - {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:9c7617df90c1365638916b98cdd9be833d31d337dbcd722485597b43c4a215bf"}, - {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:41d0f1fa4c6830176eef5b276af04c89320ea616655d01327d5ce65e50575c94"}, - {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-win32.whl", hash = "sha256:f6d3d39611ac2e4f62c3128a9eed45f19a6608670c5a2f4f07f24e8de3441d38"}, - {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-win_amd64.whl", hash = "sha256:da538167284de58a52109a9b89b8f6a53ff8437dd6dc26d33b57bf6699153122"}, - {file = "ruamel.yaml.clib-0.2.7-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:4b3a93bb9bc662fc1f99c5c3ea8e623d8b23ad22f861eb6fce9377ac07ad6072"}, - {file = "ruamel.yaml.clib-0.2.7-cp36-cp36m-macosx_12_0_arm64.whl", hash = "sha256:a234a20ae07e8469da311e182e70ef6b199d0fbeb6c6cc2901204dd87fb867e8"}, - {file = "ruamel.yaml.clib-0.2.7-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:15910ef4f3e537eea7fe45f8a5d19997479940d9196f357152a09031c5be59f3"}, - {file = "ruamel.yaml.clib-0.2.7-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:370445fd795706fd291ab00c9df38a0caed0f17a6fb46b0f607668ecb16ce763"}, - {file = "ruamel.yaml.clib-0.2.7-cp36-cp36m-win32.whl", hash = "sha256:ecdf1a604009bd35c674b9225a8fa609e0282d9b896c03dd441a91e5f53b534e"}, - {file = "ruamel.yaml.clib-0.2.7-cp36-cp36m-win_amd64.whl", hash = "sha256:f34019dced51047d6f70cb9383b2ae2853b7fc4dce65129a5acd49f4f9256646"}, - {file = "ruamel.yaml.clib-0.2.7-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:2aa261c29a5545adfef9296b7e33941f46aa5bbd21164228e833412af4c9c75f"}, - {file = "ruamel.yaml.clib-0.2.7-cp37-cp37m-macosx_12_0_arm64.whl", hash = "sha256:f01da5790e95815eb5a8a138508c01c758e5f5bc0ce4286c4f7028b8dd7ac3d0"}, - {file = "ruamel.yaml.clib-0.2.7-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:40d030e2329ce5286d6b231b8726959ebbe0404c92f0a578c0e2482182e38282"}, - {file = "ruamel.yaml.clib-0.2.7-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:c3ca1fbba4ae962521e5eb66d72998b51f0f4d0f608d3c0347a48e1af262efa7"}, - {file = "ruamel.yaml.clib-0.2.7-cp37-cp37m-win32.whl", hash = "sha256:7bdb4c06b063f6fd55e472e201317a3bb6cdeeee5d5a38512ea5c01e1acbdd93"}, - {file = "ruamel.yaml.clib-0.2.7-cp37-cp37m-win_amd64.whl", hash = "sha256:be2a7ad8fd8f7442b24323d24ba0b56c51219513cfa45b9ada3b87b76c374d4b"}, - {file = "ruamel.yaml.clib-0.2.7-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:91a789b4aa0097b78c93e3dc4b40040ba55bef518f84a40d4442f713b4094acb"}, - {file = "ruamel.yaml.clib-0.2.7-cp38-cp38-macosx_12_0_arm64.whl", hash = "sha256:99e77daab5d13a48a4054803d052ff40780278240a902b880dd37a51ba01a307"}, - {file = "ruamel.yaml.clib-0.2.7-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:3243f48ecd450eddadc2d11b5feb08aca941b5cd98c9b1db14b2fd128be8c697"}, - {file = "ruamel.yaml.clib-0.2.7-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:8831a2cedcd0f0927f788c5bdf6567d9dc9cc235646a434986a852af1cb54b4b"}, - {file = "ruamel.yaml.clib-0.2.7-cp38-cp38-win32.whl", hash = "sha256:3110a99e0f94a4a3470ff67fc20d3f96c25b13d24c6980ff841e82bafe827cac"}, - {file = "ruamel.yaml.clib-0.2.7-cp38-cp38-win_amd64.whl", hash = "sha256:92460ce908546ab69770b2e576e4f99fbb4ce6ab4b245345a3869a0a0410488f"}, - {file = "ruamel.yaml.clib-0.2.7-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5bc0667c1eb8f83a3752b71b9c4ba55ef7c7058ae57022dd9b29065186a113d9"}, - {file = "ruamel.yaml.clib-0.2.7-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:4a4d8d417868d68b979076a9be6a38c676eca060785abaa6709c7b31593c35d1"}, - {file = "ruamel.yaml.clib-0.2.7-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:bf9a6bc4a0221538b1a7de3ed7bca4c93c02346853f44e1cd764be0023cd3640"}, - {file = "ruamel.yaml.clib-0.2.7-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:a7b301ff08055d73223058b5c46c55638917f04d21577c95e00e0c4d79201a6b"}, - {file = "ruamel.yaml.clib-0.2.7-cp39-cp39-win32.whl", hash = "sha256:d5e51e2901ec2366b79f16c2299a03e74ba4531ddcfacc1416639c557aef0ad8"}, - {file = "ruamel.yaml.clib-0.2.7-cp39-cp39-win_amd64.whl", hash = "sha256:184faeaec61dbaa3cace407cffc5819f7b977e75360e8d5ca19461cd851a5fc5"}, - {file = "ruamel.yaml.clib-0.2.7.tar.gz", hash = "sha256:1f08fd5a2bea9c4180db71678e850b995d2a5f4537be0e94557668cf0f5f9497"}, + {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b42169467c42b692c19cf539c38d4602069d8c1505e97b86387fcf7afb766e1d"}, + {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-macosx_13_0_arm64.whl", hash = "sha256:07238db9cbdf8fc1e9de2489a4f68474e70dffcb32232db7c08fa61ca0c7c462"}, + {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:d92f81886165cb14d7b067ef37e142256f1c6a90a65cd156b063a43da1708cfd"}, + {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:fff3573c2db359f091e1589c3d7c5fc2f86f5bdb6f24252c2d8e539d4e45f412"}, + {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:840f0c7f194986a63d2c2465ca63af8ccbbc90ab1c6001b1978f05119b5e7334"}, + {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:024cfe1fc7c7f4e1aff4a81e718109e13409767e4f871443cbff3dba3578203d"}, + {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-win32.whl", hash = "sha256:c69212f63169ec1cfc9bb44723bf2917cbbd8f6191a00ef3410f5a7fe300722d"}, + {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-win_amd64.whl", hash = "sha256:cabddb8d8ead485e255fe80429f833172b4cadf99274db39abc080e068cbcc31"}, + {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:bef08cd86169d9eafb3ccb0a39edb11d8e25f3dae2b28f5c52fd997521133069"}, + {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-macosx_13_0_arm64.whl", hash = "sha256:b16420e621d26fdfa949a8b4b47ade8810c56002f5389970db4ddda51dbff248"}, + {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:b5edda50e5e9e15e54a6a8a0070302b00c518a9d32accc2346ad6c984aacd279"}, + {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:25c515e350e5b739842fc3228d662413ef28f295791af5e5110b543cf0b57d9b"}, + {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:46d378daaac94f454b3a0e3d8d78cafd78a026b1d71443f4966c696b48a6d899"}, + {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:09b055c05697b38ecacb7ac50bdab2240bfca1a0c4872b0fd309bb07dc9aa3a9"}, + {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-win32.whl", hash = "sha256:53a300ed9cea38cf5a2a9b069058137c2ca1ce658a874b79baceb8f892f915a7"}, + {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-win_amd64.whl", hash = "sha256:c2a72e9109ea74e511e29032f3b670835f8a59bbdc9ce692c5b4ed91ccf1eedb"}, + {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:ebc06178e8821efc9692ea7544aa5644217358490145629914d8020042c24aa1"}, + {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-macosx_13_0_arm64.whl", hash = "sha256:edaef1c1200c4b4cb914583150dcaa3bc30e592e907c01117c08b13a07255ec2"}, + {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-manylinux2014_aarch64.whl", hash = "sha256:7048c338b6c86627afb27faecf418768acb6331fc24cfa56c93e8c9780f815fa"}, + {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d176b57452ab5b7028ac47e7b3cf644bcfdc8cacfecf7e71759f7f51a59e5c92"}, + {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:3213ece08ea033eb159ac52ae052a4899b56ecc124bb80020d9bbceeb50258e9"}, + {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:aab7fd643f71d7946f2ee58cc88c9b7bfc97debd71dcc93e03e2d174628e7e2d"}, + {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-win32.whl", hash = "sha256:5c365d91c88390c8d0a8545df0b5857172824b1c604e867161e6b3d59a827eaa"}, + {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-win_amd64.whl", hash = "sha256:1758ce7d8e1a29d23de54a16ae867abd370f01b5a69e1a3ba75223eaa3ca1a1b"}, + {file = "ruamel.yaml.clib-0.2.8-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:a5aa27bad2bb83670b71683aae140a1f52b0857a2deff56ad3f6c13a017a26ed"}, + {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c58ecd827313af6864893e7af0a3bb85fd529f862b6adbefe14643947cfe2942"}, + {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-macosx_12_0_arm64.whl", hash = "sha256:f481f16baec5290e45aebdc2a5168ebc6d35189ae6fea7a58787613a25f6e875"}, + {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:3fcc54cb0c8b811ff66082de1680b4b14cf8a81dce0d4fbf665c2265a81e07a1"}, + {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:7f67a1ee819dc4562d444bbafb135832b0b909f81cc90f7aa00260968c9ca1b3"}, + {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:4ecbf9c3e19f9562c7fdd462e8d18dd902a47ca046a2e64dba80699f0b6c09b7"}, + {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:87ea5ff66d8064301a154b3933ae406b0863402a799b16e4a1d24d9fbbcbe0d3"}, + {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-win32.whl", hash = "sha256:75e1ed13e1f9de23c5607fe6bd1aeaae21e523b32d83bb33918245361e9cc51b"}, + {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-win_amd64.whl", hash = "sha256:3f215c5daf6a9d7bbed4a0a4f760f3113b10e82ff4c5c44bec20a68c8014f675"}, + {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1b617618914cb00bf5c34d4357c37aa15183fa229b24767259657746c9077615"}, + {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-macosx_12_0_arm64.whl", hash = "sha256:a6a9ffd280b71ad062eae53ac1659ad86a17f59a0fdc7699fd9be40525153337"}, + {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:665f58bfd29b167039f714c6998178d27ccd83984084c286110ef26b230f259f"}, + {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:700e4ebb569e59e16a976857c8798aee258dceac7c7d6b50cab63e080058df91"}, + {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:e2b4c44b60eadec492926a7270abb100ef9f72798e18743939bdbf037aab8c28"}, + {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:e79e5db08739731b0ce4850bed599235d601701d5694c36570a99a0c5ca41a9d"}, + {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-win32.whl", hash = "sha256:955eae71ac26c1ab35924203fda6220f84dce57d6d7884f189743e2abe3a9fbe"}, + {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-win_amd64.whl", hash = "sha256:56f4252222c067b4ce51ae12cbac231bce32aee1d33fbfc9d17e5b8d6966c312"}, + {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:03d1162b6d1df1caa3a4bd27aa51ce17c9afc2046c31b0ad60a0a96ec22f8001"}, + {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:bba64af9fa9cebe325a62fa398760f5c7206b215201b0ec825005f1b18b9bccf"}, + {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:9eb5dee2772b0f704ca2e45b1713e4e5198c18f515b52743576d196348f374d3"}, + {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:da09ad1c359a728e112d60116f626cc9f29730ff3e0e7db72b9a2dbc2e4beed5"}, + {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:184565012b60405d93838167f425713180b949e9d8dd0bbc7b49f074407c5a8b"}, + {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a75879bacf2c987c003368cf14bed0ffe99e8e85acfa6c0bfffc21a090f16880"}, + {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-win32.whl", hash = "sha256:84b554931e932c46f94ab306913ad7e11bba988104c5cff26d90d03f68258cd5"}, + {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-win_amd64.whl", hash = "sha256:25ac8c08322002b06fa1d49d1646181f0b2c72f5cbc15a85e80b4c30a544bb15"}, + {file = "ruamel.yaml.clib-0.2.8.tar.gz", hash = "sha256:beb2e0404003de9a4cab9753a8805a8fe9320ee6673136ed7f04255fe60bb512"}, ] [[package]] diff --git a/scripts/ubuntu-cuda/20-install-libcudnn.sh b/scripts/ubuntu-cuda/20-install-libcudnn.sh index 10a8666..eccaaaa 100644 --- a/scripts/ubuntu-cuda/20-install-libcudnn.sh +++ b/scripts/ubuntu-cuda/20-install-libcudnn.sh @@ -19,7 +19,7 @@ DISTRO=ubuntu${UBUNTU_RELEASE//\./} # ubuntu1804 cuda_version="cuda12.0" cudnn_version="8.8.1.*" -wget https://developer.download.nvidia.com/compute/cuda/repos/${DISTRO}/x86_64/cuda-${DISTRO}.pin +wget https://developer.download.nvidia.com/compute/cuda/repos/${DISTRO}/x86_64/cuda-${DISTRO}.pin sudo mv cuda-${DISTRO}.pin /etc/apt/preferences.d/cuda-repository-pin-600 sudo apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/${DISTRO}/x86_64/3bf863cc.pub diff --git a/scripts/ubuntu-cuda/30-install-singularity.sh b/scripts/ubuntu-cuda/30-install-singularity.sh index 059cd2d..53bd397 100644 --- a/scripts/ubuntu-cuda/30-install-singularity.sh +++ b/scripts/ubuntu-cuda/30-install-singularity.sh @@ -28,4 +28,4 @@ sudo apt-get install -y \ cd /tmp wget https://github.com/sylabs/singularity/releases/download/v3.11.1/singularity-ce_3.11.1-jammy_amd64.deb dpkg -i singularity-ce_3.11.1-jammy_amd64.deb -rm *.deb \ No newline at end of file +rm *.deb diff --git a/scripts/ubuntu-cuda/README.md b/scripts/ubuntu-cuda/README.md index e5e9016..679e019 100644 --- a/scripts/ubuntu-cuda/README.md +++ b/scripts/ubuntu-cuda/README.md @@ -3,4 +3,3 @@ ## TODO - rename to ubuntu-translations - diff --git a/scripts/ubuntu-jammy/01-kernel.sh b/scripts/ubuntu-jammy/01-kernel.sh index 164fc25..b96e243 100644 --- a/scripts/ubuntu-jammy/01-kernel.sh +++ b/scripts/ubuntu-jammy/01-kernel.sh @@ -18,7 +18,7 @@ export DEBIAN_FRONTEND=noninteractive retry apt-get update # TODO: figure out issue on jammy -# error: +# error: # The following packages have unmet dependencies: # linux-crashdump : Depends: kdump-tools but it is not installable # diff --git a/scripts/worker-runner-gw-systemd/10-create-worker-runner-config.sh b/scripts/worker-runner-gw-systemd/10-create-worker-runner-config.sh index 248306e..26735c2 100644 --- a/scripts/worker-runner-gw-systemd/10-create-worker-runner-config.sh +++ b/scripts/worker-runner-gw-systemd/10-create-worker-runner-config.sh @@ -19,4 +19,4 @@ worker: implementation: generic-worker path: /home/ubuntu/generic_worker/generic-worker configPath: /home/ubuntu/generic_worker/generic-worker.config -EOF \ No newline at end of file +EOF diff --git a/scripts/worker-runner-gw-systemd/30-create-tc-job-dirs.sh b/scripts/worker-runner-gw-systemd/30-create-tc-job-dirs.sh index 12daf5f..764c613 100644 --- a/scripts/worker-runner-gw-systemd/30-create-tc-job-dirs.sh +++ b/scripts/worker-runner-gw-systemd/30-create-tc-job-dirs.sh @@ -13,4 +13,4 @@ USER_DIR="/home/ubuntu" DIR_LIST="${USER_DIR}/tasks ${USER_DIR}/caches ${USER_DIR}/downloads" sudo mkdir -p ${DIR_LIST} -sudo chown -R ubuntu:ubuntu ${DIR_LIST} \ No newline at end of file +sudo chown -R ubuntu:ubuntu ${DIR_LIST} diff --git a/scripts/worker-runner-gw-systemd/README.md b/scripts/worker-runner-gw-systemd/README.md index daf4d74..6a5a1ad 100644 --- a/scripts/worker-runner-gw-systemd/README.md +++ b/scripts/worker-runner-gw-systemd/README.md @@ -2,7 +2,7 @@ This scripts directory configures worker-runner and generic-worker. -TODO: +TODO: - move this to generic-worker-linux? ## Inputs @@ -20,4 +20,4 @@ TODO: ## Notes -none yet \ No newline at end of file +none yet diff --git a/tests/test_secrets.py b/tests/test_secrets.py index 6c45d05..4db44bf 100644 --- a/tests/test_secrets.py +++ b/tests/test_secrets.py @@ -3,17 +3,23 @@ from monopacker.secrets import pack_secrets + def test_pack_secrets(fs): # "fs" is the reference to the fake file system - fs.create_file('secrets.yaml', contents=json.dumps([ - {'name': 'a-secret', 'path': '/path/to/my/secret', 'value': 'sshh'}, - {'name': 'b-secret', 'path': '/path/to/my/stuff', 'value': 'quiet'}, - ])) - pack_secrets('secrets.yaml', 'secrets.tar') - with tarfile.open('secrets.tar') as tar: + fs.create_file( + "secrets.yaml", + contents=json.dumps( + [ + {"name": "a-secret", "path": "/path/to/my/secret", "value": "sshh"}, + {"name": "b-secret", "path": "/path/to/my/stuff", "value": "quiet"}, + ] + ), + ) + pack_secrets("secrets.yaml", "secrets.tar") + with tarfile.open("secrets.tar") as tar: print(tar.getnames()) # note that initial `/` is stripped - ti = tar.extractfile('path/to/my/secret') - assert ti.read() == b'sshh' - ti = tar.extractfile('path/to/my/stuff') - assert ti.read() == b'quiet' + ti = tar.extractfile("path/to/my/secret") + assert ti.read() == b"sshh" + ti = tar.extractfile("path/to/my/stuff") + assert ti.read() == b"quiet" diff --git a/tests/test_template_packer.py b/tests/test_template_packer.py index d484138..4570403 100644 --- a/tests/test_template_packer.py +++ b/tests/test_template_packer.py @@ -9,6 +9,7 @@ generate_packer_template, ) + def test_merge_vars(): base = {"abc": 123, "foo": "bar", "blah": {"sub_foo": "sub_bar", "cow": "quack"}} override = {"abc": 456, "blah": {"cow": "moo"}} @@ -34,7 +35,7 @@ def test_merge_vars_overwrite_arrays(): def test_merge_vars_merge_sub_sub_dicts(): - base = {"combined": {"b": {1: 10, 2:20}}} + base = {"combined": {"b": {1: 10, 2: 20}}} override = {"combined": {"b": {2: 120, 3: 130}}} expected = {"combined": {"b": {1: 10, 2: 120, 3: 130}}} @@ -68,7 +69,10 @@ def test_get_builders_for_templating(tmpdir): # tests the normal case, builder with var_files, vars builders = get_builders_for_templating( - ["foo"], builders_dir=builders_dir, var_files_dir=vars_dir, scripts_dir=scripts_dir, + ["foo"], + builders_dir=builders_dir, + var_files_dir=vars_dir, + scripts_dir=scripts_dir, ) assert builders[0]["template"] == "foo.yaml" @@ -94,7 +98,10 @@ def test_get_builders_for_templating(tmpdir): # test that we get FileNotFoundError when passing a nonexistant subdir with pytest.raises(FileNotFoundError, match=r".*bar.yaml.*"): builders = get_builders_for_templating( - ["bar"], builders_dir=builders_dir, var_files_dir=vars_dir, scripts_dir=scripts_dir, + ["bar"], + builders_dir=builders_dir, + var_files_dir=vars_dir, + scripts_dir=scripts_dir, ) @@ -139,38 +146,54 @@ def test_generate_packer_template(tmpdir): files_dir = tmpdir.mkdir("files") secrets_file = tmpdir.join("secrets.yml") - builders_dir.join("linux.yaml").write(json.dumps({ - "template": "alibaba_linux", - "platform": "linux", - "builder_var_files": ["bv", "env"], - "script_directories": ["facebook-worker"], - "builder_vars": { - "execute_command": "do-it", - "ssh_timeout": "30m", - }, - })) - - builders_dir.join("winny.yaml").write(json.dumps({ - "template": "openstack_windows", - "platform": "windows", - "builder_var_files": [], - "script_directories": ["win-worker"], - "builder_vars": { - "execute_command": "do-it", - "ssh_timeout": "30m", - }, - })) - - templates_dir.join("alibaba_linux.jinja2").write(textwrap.dedent("""\ + builders_dir.join("linux.yaml").write( + json.dumps( + { + "template": "alibaba_linux", + "platform": "linux", + "builder_var_files": ["bv", "env"], + "script_directories": ["facebook-worker"], + "builder_vars": { + "execute_command": "do-it", + "ssh_timeout": "30m", + }, + } + ) + ) + + builders_dir.join("winny.yaml").write( + json.dumps( + { + "template": "openstack_windows", + "platform": "windows", + "builder_var_files": [], + "script_directories": ["win-worker"], + "builder_vars": { + "execute_command": "do-it", + "ssh_timeout": "30m", + }, + } + ) + ) + + templates_dir.join("alibaba_linux.jinja2").write( + textwrap.dedent( + """\ - name: a packer builder type: alibaba a-is: {{builder.vars.a}} - """)) + """ + ) + ) - templates_dir.join("openstack_windows.jinja2").write(textwrap.dedent("""\ + templates_dir.join("openstack_windows.jinja2").write( + textwrap.dedent( + """\ - name: a packer builder type: openstack - """)) + """ + ) + ) secrets_file.write(json.dumps([])) @@ -178,16 +201,24 @@ def test_generate_packer_template(tmpdir): scripts_dir.mkdir("win-worker").join("01-win.ps1").write("ECHO hello") - var_files_dir.join("bv.yaml").write(json.dumps({ - "a": 10, - "b": 20, - })) + var_files_dir.join("bv.yaml").write( + json.dumps( + { + "a": 10, + "b": 20, + } + ) + ) - var_files_dir.join("env.yaml").write(json.dumps({ - "env_vars": { - "AN_ENV_VAR": 'env!', - }, - })) + var_files_dir.join("env.yaml").write( + json.dumps( + { + "env_vars": { + "AN_ENV_VAR": "env!", + }, + } + ) + ) packer_template = generate_packer_template( builders=["linux", "winny"], @@ -199,85 +230,84 @@ def test_generate_packer_template(tmpdir): secrets_file=str(secrets_file), ) - assert(packer_template == { - 'builders': [ + assert packer_template == { + "builders": [ { - 'name': 'a packer builder', - 'type': 'alibaba', - 'a-is': 10, + "name": "a packer builder", + "type": "alibaba", + "a-is": 10, }, { - 'name': 'a packer builder', - 'type': 'openstack', + "name": "a packer builder", + "type": "openstack", }, ], - 'provisioners': [ + "provisioners": [ { - 'type': 'file', - 'source': './files.tar', - 'destination': '/tmp/', + "type": "file", + "source": "./files.tar", + "destination": "/tmp/", }, { - 'type': 'shell', - 'inline': [ - 'sudo tar xvf /tmp/files.tar -C / --strip-components=1', - 'rm /tmp/files.tar', + "type": "shell", + "inline": [ + "sudo tar xvf /tmp/files.tar -C / --strip-components=1", + "rm /tmp/files.tar", ], }, { - 'type': 'file', - 'source': './secrets.tar', - 'destination': '/tmp/', + "type": "file", + "source": "./secrets.tar", + "destination": "/tmp/", }, { - 'type': 'shell', - 'inline': [ - 'sudo mkdir -p /etc/taskcluster/secrets', - 'sudo tar xvf /tmp/secrets.tar -C /', - 'sudo chown root:root -R /etc/taskcluster', - 'sudo chmod 0400 -R /etc/taskcluster/secrets', - 'rm /tmp/secrets.tar', + "type": "shell", + "inline": [ + "sudo mkdir -p /etc/taskcluster/secrets", + "sudo tar xvf /tmp/secrets.tar -C /", + "sudo chown root:root -R /etc/taskcluster", + "sudo chmod 0400 -R /etc/taskcluster/secrets", + "rm /tmp/secrets.tar", ], - 'only': ['linux'], + "only": ["linux"], }, { - 'type': 'shell', - 'inline': [ - '/usr/bin/cloud-init status --wait', + "type": "shell", + "inline": [ + "/usr/bin/cloud-init status --wait", ], - 'only': ['linux'], + "only": ["linux"], }, { - 'type': 'shell', - 'scripts': [str(scripts_dir.join("facebook-worker", "01-fb.sh"))], - 'environment_vars': ["AN_ENV_VAR=env!"], - 'execute_command': "do-it", - 'expect_disconnect': True, - 'start_retry_timeout': '30m', - 'only': ['linux'], + "type": "shell", + "scripts": [str(scripts_dir.join("facebook-worker", "01-fb.sh"))], + "environment_vars": ["AN_ENV_VAR=env!"], + "execute_command": "do-it", + "expect_disconnect": True, + "start_retry_timeout": "30m", + "only": ["linux"], }, { - 'type': 'powershell', - 'scripts': [str(scripts_dir.join("facebook-worker", "01-fb.sh"))], - 'only': [], + "type": "powershell", + "scripts": [str(scripts_dir.join("facebook-worker", "01-fb.sh"))], + "only": [], }, { - 'type': 'shell', - 'scripts': [str(scripts_dir.join("win-worker", "01-win.ps1"))], - 'environment_vars': None, - 'execute_command': "do-it", - 'expect_disconnect': True, - 'start_retry_timeout': '30m', - 'only': [], + "type": "shell", + "scripts": [str(scripts_dir.join("win-worker", "01-win.ps1"))], + "environment_vars": None, + "execute_command": "do-it", + "expect_disconnect": True, + "start_retry_timeout": "30m", + "only": [], }, { - 'type': 'powershell', - 'scripts': [str(scripts_dir.join("win-worker", "01-win.ps1"))], - 'only': ['winny'], - } + "type": "powershell", + "scripts": [str(scripts_dir.join("win-worker", "01-win.ps1"))], + "only": ["winny"], + }, ], - 'post-processors': [ - {'type': 'manifest', 'output': 'packer-artifacts.json', 'strip_path': True}, + "post-processors": [ + {"type": "manifest", "output": "packer-artifacts.json", "strip_path": True}, ], - }) - + }