add pre-commit

.github/workflows/github-actions.yml

@@ -14,7 +14,7 @@ env:
 jobs:
   pytest:
     runs-on: ubuntu-latest
-    steps:    
+    steps:
       - name: Checkout
         uses: actions/checkout@v3
       - name: Set up Python ${{ env.PYTHON_VERSION }}
@@ -42,11 +42,11 @@ jobs:
         run: poetry install
       - name: pip list
         run: |
-          poetry run pip list          
+          poetry run pip list
       - name: Run tests
         run: |
           poetry run python -m pytest -sxv
-  poetry-update-check:
+  pre-commit-check:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
@@ -68,29 +68,58 @@ jobs:
         with:
           path: ~/.local
           key: poetry-local-${{ runner.os }}-${{ steps.setup_python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-${{ hashFiles('.github/workflows/*.yml') }}
-      - name: Install Poetry ${{ env.POETRY_VERSION }}
-        run: |
-          curl -sSL ${{ env.POETRY_URL }} | python - --version ${{ env.POETRY_VERSION }}
-          echo "$HOME/.local/bin" >> $GITHUB_PATH
-      - name: Install Dependencies
-        run: poetry install
-      - name: pip list
-        run: |
-          poetry run pip list                    
-      - name: Run tests
-        run: |
-          poetry run python -m pytest -sxv
-      - name: poetry update
-        run: |
-          poetry update
-      - name: check if changes
-        run: |
-            git status --porcelain
-            if ! output=$(git status --porcelain) || [ -n "$output" ]; then
-                echo "*** 'poetry update' produced changes to the repository; these changes should be checked in ***";
-                git --no-pager diff;
-                exit 1;
-            fi                       
+      # TODO: just use poetry to install pre-commit? chicken before egg situation...
+      - name: Install pre-commit
+        run: pip3 install pre-commit
+      - name: Run pre-commit
+        # skip poetry-lock because it breaks (aka changes) when run on different platforms
+        #   - the pre-commit on developer's hosts should catch if it's out of date (the goal)
+        run: SKIP=poetry-lock pre-commit run -av
+  # poetry-update-check:
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - name: Checkout
+  #       uses: actions/checkout@v3
+  #     - name: Set up Python ${{ env.PYTHON_VERSION }}
+  #       uses: actions/setup-python@v4
+  #       with:
+  #         python-version: ${{ env.PYTHON_VERSION }}
+  #       id: setup_python
+  #     # Poetry cache depends on OS, Python version and Poetry version.
+  #     - name: Cache Poetry cache
+  #       uses: actions/cache@v3
+  #       with:
+  #         path: ~/.cache/pypoetry
+  #         key: poetry-cache-${{ runner.os }}-${{ steps.setup_python.outputs.python-version }}-${{ env.POETRY_VERSION }}
+  #     # virtualenv cache should depends on OS, Python version and `poetry.lock` (and optionally workflow files).
+  #     - name: Cache Packages
+  #       uses: actions/cache@v3
+  #       with:
+  #         path: ~/.local
+  #         key: poetry-local-${{ runner.os }}-${{ steps.setup_python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-${{ hashFiles('.github/workflows/*.yml') }}
+  #     - name: Install Poetry ${{ env.POETRY_VERSION }}
+  #       run: |
+  #         curl -sSL ${{ env.POETRY_URL }} | python - --version ${{ env.POETRY_VERSION }}
+  #         echo "$HOME/.local/bin" >> $GITHUB_PATH
+  #     - name: Install Dependencies
+  #       run: poetry install
+  #     - name: pip list
+  #       run: |
+  #         poetry run pip list
+  #     - name: Run tests
+  #       run: |
+  #         poetry run python -m pytest -sxv
+  #     - name: poetry update
+  #       run: |
+  #         poetry update
+  #     - name: check if changes
+  #       run: |
+  #           git status --porcelain
+  #           if ! output=$(git status --porcelain) || [ -n "$output" ]; then
+  #               echo "*** 'poetry update' produced changes to the repository; these changes should be checked in ***";
+  #               git --no-pager diff;
+  #               exit 1;
+  #           fi
   monopacker-validate:
     runs-on: ubuntu-latest
     steps:
@@ -100,7 +129,7 @@ jobs:
         uses: hashicorp/setup-packer@main
         id: setup
         with:
-          version: ${{ env.PACKER_VERSION }}        
+          version: ${{ env.PACKER_VERSION }}
       - name: Set up Python ${{ env.PYTHON_VERSION }}
         uses: actions/setup-python@v4
         with:
@@ -134,4 +163,4 @@ jobs:
                     poetry run monopacker validate "$builder_instance"
                 done
           echo ""
-          echo "ALL VALIDATED"
+          echo "ALL VALIDATED"

.pre-commit-config.yaml

@@ -0,0 +1,21 @@
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.5.0
+    hooks:
+    -   id: check-yaml
+    -   id: end-of-file-fixer
+    -   id: trailing-whitespace
+-   repo: https://github.com/psf/black
+    rev: 23.11.0
+    hooks:
+    -   id: black
+-   repo: https://github.com/python-poetry/poetry
+    rev: '8dbe41e'
+    hooks:
+    -   id: poetry-check
+        args: ["--lock"]
+    -   id: poetry-lock
+-   repo: https://github.com/aerickson/poetry
+    rev: '0113a0c'
+    hooks:
+    -   id: poetry-update

LICENSE

@@ -351,4 +351,3 @@ Exhibit B - “Incompatible With Secondary Licenses” Notice
       This Source Code Form is “Incompatible
       With Secondary Licenses”, as defined by
       the Mozilla Public License, v. 2.0.
-

README.md

@@ -124,4 +124,4 @@ Mostly, I just haven't tried to make this work.
 
 To run the tests for this library, run `poetry run pytest`.
 
-To update dependencies, run `poetry update`.
+To update dependencies, run `poetry update`.

bin/monopacker

@@ -2,4 +2,5 @@
 
 if __name__ == "__main__":
     from monopacker.main import main
+
     main()

monopacker/files.py

@@ -3,7 +3,8 @@
 import os, sys, tarfile, tempfile
 from pathlib import Path
 
+
 def pack_files(files_dir, files_tar):
     # create a directory with `files` as the top level directory component.
     with tarfile.open(files_tar, "w") as tar:
-        tar.add(files_dir, arcname='files')
+        tar.add(files_dir, arcname="files")

monopacker/main.py

@@ -11,13 +11,15 @@
     generate_packer_template_params,
 )
 
+
 @click.group()
 def main():
     """
     Manages building worker images for Taskcluster.
     """
 
-@main.command(name='packer-template')
+
+@main.command(name="packer-template")
 @generate_packer_template_params
 def packer_template(**kwargs):
     """This tool expects a jinja2 templated packer.yaml and the name of one or more builders.
@@ -27,18 +29,20 @@ def packer_template(**kwargs):
     packer_template = generate_packer_template(**kwargs)
     print(json.dumps(packer_template, sort_keys=True, indent=4))
 
-@main.command(name='validate')
+
+@main.command(name="validate")
 @generate_packer_template_params
 @run_packer_params
 def validate(**kwargs):
     """Validate the generated template with 'packer validate'"""
     packer_template = generate_packer_template(**kwargs)
-    run_packer('validate', packer_template, **kwargs)
+    run_packer("validate", packer_template, **kwargs)
+
 
-@main.command(name='build')
+@main.command(name="build")
 @generate_packer_template_params
 @run_packer_params
 def validate(**kwargs):
     """Validate the generated template with 'packer validate'"""
     packer_template = generate_packer_template(**kwargs)
-    run_packer('build', packer_template, **kwargs)
+    run_packer("build", packer_template, **kwargs)

monopacker/packer.py

@@ -5,33 +5,35 @@
 import json
 import shlex
 
+
 def run_packer_params(fn):
     "Decorate a click function with options for run_packer"
     params = [
         click.option(
-            "--packer",
-            type=str,
-            help="path to the packer binary",
-            default='packer'),
+            "--packer", type=str, help="path to the packer binary", default="packer"
+        ),
         click.option(
             "--packer-args",
             type=str,
             help="additional arguments to pass to packer (shell-quoted)",
-            default='')
-        ]
+            default="",
+        ),
+    ]
     params.reverse()
     for param in params:
         fn = param(fn)
     return fn
 
+
 def run_packer(command, input, *, packer, packer_args, **_):
     """Run packer with the given (JSON) template, passing additional extra_args"""
     packer_args = shlex.split(packer_args)
     try:
         subprocess.run(
-            [packer, command] + packer_args + ['-'],
-            input=bytes(json.dumps(input), 'utf8'),
-            check=True)
+            [packer, command] + packer_args + ["-"],
+            input=bytes(json.dumps(input), "utf8"),
+            check=True,
+        )
         return True
     except subprocess.CalledProcessError as e:
         print(f"packer exited with code {e.returncode}", file=sys.stderr)

monopacker/secrets.py

@@ -7,6 +7,7 @@
 
 yaml = YAML(typ="safe")
 
+
 def pack_secrets(secrets_file, secrets_tar):
     with open(secrets_file, "r") as f:
         secrets = yaml.load(f)
@@ -22,7 +23,7 @@ def pack_secrets(secrets_file, secrets_tar):
                     raise RuntimeError(f"Encountered secret {name} without `path` key")
                 if "value" not in secret:
                     raise RuntimeError(f"Encountered secret {name} without `value` key")
-                path = secret["path"].lstrip('/')
+                path = secret["path"].lstrip("/")
                 value = bytes(secret["value"], "utf8")
                 ti = tarfile.TarInfo(path)
                 ti.size = len(value)