Add staging to selftest (#127)

* test: Add marker "staging"

Only run the marked tests when "--staging" is used. This allows us
* enable only the tests we know work on staging
* slowly add support to more tests (likely by adding staging test assets)
* at some point switch the marker use so that default is staging support
  and the remaining tests are marked with skipif()

I've only enabled three tests: they seem like they should work on
staging, I am not sure about any others.

Signed-off-by: Jussi Kukkonen <jkukkonen@google.com>

* action.py: Support GHA_SIGSTORE_CONFORMANCE_ENABLE_STAGING

This variable results in the test suite being run twice:
* first without --staging
* then with --staging

Signed-off-by: Jussi Kukkonen <jkukkonen@google.com>

* action: Add new input "enable-staging"

Input defaults to false for now: We may want to consider enabling
by default when we release (or requiring a value).

Signed-off-by: Jussi Kukkonen <jkukkonen@google.com>

* self-test: Enable staging test

Signed-off-by: Jussi Kukkonen <jkukkonen@google.com>

* Action: Tweak output

* Single summary output line
* Add line in log with the "staging"/"production": this is useful to
  tell apart the two pytest result blocks

Signed-off-by: Jussi Kukkonen <jkukkonen@google.com>

* README: Update the action usage section

The example had bit-rotted a bit so I rewrote it and added
enable-staging and xfail to README.

Signed-off-by: Jussi Kukkonen <jkukkonen@google.com>

---------

Signed-off-by: Jussi Kukkonen <jkukkonen@google.com>

.github/workflows/conformance.yml

@@ -25,4 +25,5 @@ jobs:
         id: sigstore-conformance
         with:
           entrypoint: ${{ github.workspace }}/sigstore-python-conformance
+          enable-staging: true
           xfail: "test_verify_with_trust_root test_verify_dsse_bundle_with_trust_root"

README.md

@@ -34,28 +34,34 @@ Some general testing principles for this suite are:
 
 ## Usage
 
-Simply add `sigstore/sigstore-conformance` to one of your workflows:
-
-```yaml
-jobs:
-  conformance:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: install
-        run: python -m pip install .
-      - uses: sigstore/sigstore-conformance@v0.0.10
-        with:
-          entrypoint: sigstore
-```
-
-The only required configuration is the `entrypoint` parameter which provides a
-command to invoke the client. `sigstore-conformance` expects that the client
-exposes a CLI that conforms to the protocol outlined [here](docs/cli_protocol.md).
-
-In the example above, the workflow is installing [sigstore-python](https://github.com/sigstore/sigstore-python)
-and providing `sigstore` as the `entrypoint` since this is the command used to
-invoke the client.
+1. Include an executable in your project that implements the
+client-under-test [CLI protocol](docs/cli_protocol.md).
+2. Use the `sigstore/sigstore-conformance` action in your test workflow:
+    ```yaml
+    jobs:
+      conformance:
+        runs-on: ubuntu-latest
+        steps:
+          - uses: actions/checkout@v4
+          # insert your client installation steps here
+          - uses: sigstore/sigstore-conformance@v0.0.10
+            with:
+              entrypoint: my-conformance-client
+    ```
+
+See [sigstore-python conformance test](https://github.com/sigstore/sigstore-python/blob/main/.github/workflows/conformance.yml)
+for a complete example.
+
+### `sigstore/sigstore-conformance` action inputs
+
+The important action inputs are
+* `entrypoint`: required string. A command that implements the client-under-test
+  [CLI protocol](docs/cli_protocol.md)
+* `enable-staging`: optional boolean. When true, the test suite will run tests against
+  staging infrastructure in addition to running them against production infrastructure
+* `xfail`: optional string. Whitespace separated test names that are expected to fail.
+
+See [action.yml](action.yml) for full list of inputs.
 
 ## Development
 

action.py

@@ -14,6 +14,7 @@
 _RENDER_SUMMARY = os.getenv("GHA_SIGSTORE_CONFORMANCE_SUMMARY", "true") == "true"
 _DEBUG = os.getenv("GHA_SIGSTORE_CONFORMANCE_INTERNAL_BE_CAREFUL_DEBUG", "false") != "false"
 _ACTION_PATH = Path(os.getenv("GITHUB_ACTION_PATH"))  # type: ignore
+_ENABLE_STAGING = os.getenv("GHA_SIGSTORE_CONFORMANCE_ENABLE_STAGING", "false").lower() == "true"
 
 
 def _summary(msg):
@@ -26,35 +27,42 @@ def _debug(msg):
         print(f"\033[93mDEBUG: {msg}\033[0m", file=sys.stderr)
 
 
-def _sigstore_conformance(*args) -> int:
-    return pytest.main([str(_ACTION_PATH / "test"), *args])
+def _sigstore_conformance(staging: bool) -> int:
+    args = []
 
+    if _DEBUG:
+        args.extend(["-s", "-vv", "--showlocals"])
 
-sigstore_conformance_args = []
+    entrypoint = os.getenv("GHA_SIGSTORE_CONFORMANCE_ENTRYPOINT")
+    if entrypoint:
+        args.extend(["--entrypoint", entrypoint])
 
-if _DEBUG:
-    sigstore_conformance_args.extend(["-s", "-vv", "--showlocals"])
+    if staging:
+        args.append("--staging")
 
-entrypoint = os.getenv("GHA_SIGSTORE_CONFORMANCE_ENTRYPOINT")
-if entrypoint:
-    sigstore_conformance_args.extend(["--entrypoint", entrypoint])
+    skip_signing = os.getenv("GHA_SIGSTORE_CONFORMANCE_SKIP_SIGNING", "false").lower() == "true"
+    if skip_signing:
+        args.extend(["--skip-signing"])
 
-skip_signing = os.getenv("GHA_SIGSTORE_CONFORMANCE_SKIP_SIGNING", "false").lower() == "true"
-if skip_signing:
-    sigstore_conformance_args.extend(["--skip-signing"])
+    gh_token = os.getenv("GHA_SIGSTORE_GITHUB_TOKEN")
+    if gh_token:
+        args.extend(["--github-token", gh_token])
 
-gh_token = os.getenv("GHA_SIGSTORE_GITHUB_TOKEN")
-if gh_token:
-    sigstore_conformance_args.extend(["--github-token", gh_token])
+    infra = "staging" if staging else "production"
+    print(f"running sigstore-conformance against Sigstore {infra} infrastructure")
+    _debug(f"running: sigstore-conformance {[str(a) for a in args]}")
 
-_debug(f"running: sigstore-conformance {[str(a) for a in sigstore_conformance_args]}")
+    return pytest.main([str(_ACTION_PATH / "test"), *args])
 
-status = _sigstore_conformance(*sigstore_conformance_args)
+
+# Run against production, then optionally against staging
+status = _sigstore_conformance(staging=False)
+if _ENABLE_STAGING:
+    status += _sigstore_conformance(staging=True)
 
 if status == 0:
     _summary("🎉 sigstore-conformance exited successfully")
 else:
     _summary("❌ sigstore-conformance found one or more test failures")
 
-
 sys.exit(status)

action.yml

@@ -15,6 +15,10 @@ inputs:
     description: "skip tests that involve signing (default false)"
     required: false
     default: "false"
+  enable-staging:
+    description: "Test against staging infrastructure as well as production (default false)"
+    required: false
+    default: "false"
   xfail:
     description: "one or more tests that are expected to fail, whitespace-separated"
     required: false
@@ -34,6 +38,7 @@ runs:
       run: |
         ${{ github.action_path }}/action.py
       env:
+        GHA_SIGSTORE_CONFORMANCE_ENABLE_STAGING: "${{ inputs.enable-staging }}"
         GHA_SIGSTORE_CONFORMANCE_ENTRYPOINT: "${{ inputs.entrypoint }}"
         GHA_SIGSTORE_CONFORMANCE_INTERNAL_BE_CAREFUL_DEBUG: "${{ inputs.internal-be-careful-debug }}"
         GHA_SIGSTORE_CONFORMANCE_SKIP_SIGNING: "${{ inputs.skip-signing }}"

test/conftest.py

@@ -73,13 +73,16 @@ def pytest_addoption(parser) -> None:
 def pytest_runtest_setup(item):
     if "signing" in item.keywords and item.config.getoption("--skip-signing"):
         pytest.skip("skipping test that requires signing support due to `--skip-signing` flag")
+    if "staging" not in item.keywords and item.config.getoption("--staging"):
+        pytest.skip("skipping test that does not support staging yet due to `--staging` flag")
 
 
 def pytest_configure(config):
     if not config.getoption("--github-token") and not config.getoption("--skip-signing"):
         raise ConfigError("Please specify one of '--github-token' or '--skip-signing'")
 
     config.addinivalue_line("markers", "signing: mark test as requiring signing functionality")
+    config.addinivalue_line("markers", "staging: mark test as supporting testing against staging")
 
 
 def pytest_internalerror(excrepr, excinfo):

test/test_signature_verify.py

@@ -7,6 +7,7 @@
 
 
 @pytest.mark.signing
+@pytest.mark.staging
 def test_verify_empty(client: SigstoreClient, make_materials: _MakeMaterials) -> None:
     """
     Tests that verification fails with empty artifacts, certificates and
@@ -34,6 +35,7 @@ def test_verify_empty(client: SigstoreClient, make_materials: _MakeMaterials) ->
 
 
 @pytest.mark.signing
+@pytest.mark.staging
 def test_verify_mismatch(client: SigstoreClient, make_materials: _MakeMaterials) -> None:
     """
     Tests that verification fails with mismatching artifacts, certificates and

test/test_simple.py

@@ -6,6 +6,7 @@
 
 
 @pytest.mark.signing
+@pytest.mark.staging
 def test_simple(client: SigstoreClient, make_materials: _MakeMaterials) -> None:
     """
     A simple test that signs and verifies an artifact for a given Sigstore