From 1f14f162b117817813c760b56d41a47732d3f181 Mon Sep 17 00:00:00 2001 From: Jingwei Wang <48091236+sky1122@users.noreply.github.com> Date: Fri, 10 Jan 2025 17:39:03 -0600 Subject: [PATCH] ci: add generate log files and upload to S3 bucket (#1194) Signed-off-by: Jingwei Wang --- .github/workflows/build-and-test-msi.yaml | 10 +-- .github/workflows/e2e-linux.yaml | 52 +++++++++++++- .github/workflows/e2e-macos.yaml | 82 ++++++++++++++++++++- .github/workflows/e2e-windows.yaml | 87 ++++++++++++++++++++++- .github/workflows/test-pkg.yaml | 8 +-- Makefile | 33 +++++++-- 6 files changed, 253 insertions(+), 19 deletions(-) diff --git a/.github/workflows/build-and-test-msi.yaml b/.github/workflows/build-and-test-msi.yaml index 70ed45301..2b2790b36 100644 --- a/.github/workflows/build-and-test-msi.yaml +++ b/.github/workflows/build-and-test-msi.yaml @@ -153,7 +153,7 @@ jobs: make clean cd deps/finch-core && make clean exit 0 # Cleanup may set the exit code e.g. if a file doesn't exist; just ignore - + msi-e2e-tests: needs: - get-tag-name @@ -232,11 +232,11 @@ jobs: # set path to use newer ssh version $newPath = (";C:\Program Files\Git\bin\;" + "C:\Program Files\Git\usr\bin\;" + "$env:Path") $env:Path = $newPath - + git status git clean -f -d $env:INSTALLED="true" - make test-e2e-vm + make test-e2e-vm-with-report - name: Remove Finch VM timeout-minutes: 2 shell: pwsh @@ -250,11 +250,11 @@ jobs: # set path to use newer ssh version $newPath = (";C:\Program Files\Git\bin\;" + "C:\Program Files\Git\usr\bin\;" + "$env:Path") $env:Path = $newPath - + git status git clean -f -d $env:INSTALLED="true" - make test-e2e-container + make test-e2e-container-with-report - name: Uninstall Finch silently if: ${{ always() }} run: | diff --git a/.github/workflows/e2e-linux.yaml b/.github/workflows/e2e-linux.yaml index ee3d3c1ff..3faf8bd24 100644 --- a/.github/workflows/e2e-linux.yaml +++ b/.github/workflows/e2e-linux.yaml @@ -60,7 +60,7 @@ jobs: - name: configure aws credentials uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2 # this action requires node20, skip on AL2 - if: ${{ steps.vars.outputs.has_creds == true && (!(startsWith(inputs.os, 'amazon') && inputs.version == '2' ))}} + if: ${{ steps.vars.outputs.has_creds == 'true' && (!(startsWith(inputs.os, 'amazon') && inputs.version == '2' ))}} with: role-to-assume: ${{ secrets.ROLE }} role-session-name: credhelper-test @@ -90,14 +90,60 @@ jobs: sudo systemctl start finch.service sudo systemctl start finch-buildkit.service sudo systemctl start finch-soci.service + - name: Set up REPORT_DIR + run: | + echo "REPORT_DIR=${{ github.workspace }}/reports" >> $GITHUB_ENV - name: Run e2e tests run: | git status git clean -f -d # required by one of the tests which uses SSH_AUTH_SOCK eval "$(ssh-agent -s)" - INSTALLED=true REGISTRY=${{ steps.vars.outputs.has_creds == true && env.REGISTRY || '' }} sudo -E make test-e2e-container - INSTALLED=true REGISTRY=${{ steps.vars.outputs.has_creds == true && env.REGISTRY || '' }} sudo -E make test-e2e-vm + if [ "${{ github.event_name }}" == "push" ] && [ "${{ github.ref }}" == "refs/heads/main" ] || [ "${{ github.event_name }}" == "pull_request" ] && [ "${{ github.base_ref }}" == "main" ]; then + INSTALLED=true REGISTRY=${{ steps.vars.outputs.has_creds == true && env.REGISTRY || '' }} sudo -E make test-e2e-container-with-report + INSTALLED=true REGISTRY=${{ steps.vars.outputs.has_creds == true && env.REGISTRY || '' }} sudo -E make test-e2e-vm-with-report + else + INSTALLED=true REGISTRY=${{ steps.vars.outputs.has_creds == true && env.REGISTRY || '' }} sudo -E make test-e2e-container + INSTALLED=true REGISTRY=${{ steps.vars.outputs.has_creds == true && env.REGISTRY || '' }} sudo -E make test-e2e-vm + fi + - name: Change ownership of reports + if: always() + run: | + if [ ! -d "$REPORT_DIR" ]; then + echo "Error: Directory $REPORT_DIR does not exist." + exit 1 + fi + + USER=$(whoami) + GROUP=$(id -gn) + + if sudo chown -R "$USER:$GROUP" "$REPORT_DIR"; then + echo "Ownership of $REPORT_DIR changed to $USER:$GROUP" + else + echo "Error: Failed to change ownership of $REPORT_DIR" + exit 1 + fi + - name: Check and upload e2e tests reports to s3 bucket + if: ${{ steps.vars.outputs.has_creds == 'true' && (!(startsWith(inputs.os, 'amazon') && inputs.version == '2' ))}} + run: | + VM_REPORT="${{ github.workspace }}/reports/${{ github.run_id }}-${{ github.run_attempt }}-e2e-vm-report.json" + CONTAINER_REPORT="${{ github.workspace }}/reports/${{ github.run_id }}-${{ github.run_attempt }}-e2e-container-report.json" + + if [ -f "$VM_REPORT" ]; then + echo "VM report file exists. Uploading to S3..." + aws s3 cp "$VM_REPORT" "s3://finch-e2e-test-log-reports/${{ inputs.os }}-${{ inputs.arch }}/${{ github.run_id }}-${{ github.run_attempt }}-e2e-vm-report.json" + echo "VM report uploaded successfully." + else + echo "VM report file does not exist. Skipping upload." + fi + + if [ -f "$CONTAINER_REPORT" ]; then + echo "Container report file exists. Uploading to S3..." + aws s3 cp "$CONTAINER_REPORT" "s3://finch-e2e-test-log-reports/${{ inputs.os }}-${{ inputs.arch }}/${{ github.run_id }}-${{ github.run_attempt }}-e2e-container-report.json" + echo "Container report uploaded successfully." + else + echo "Container report file does not exist. Skipping upload." + fi - name: Clean up repo AL2 if: ${{ (startsWith(inputs.os, 'amazon') && inputs.version == '2' && always() ) }} run: | diff --git a/.github/workflows/e2e-macos.yaml b/.github/workflows/e2e-macos.yaml index 51c875af9..0129188d2 100644 --- a/.github/workflows/e2e-macos.yaml +++ b/.github/workflows/e2e-macos.yaml @@ -35,6 +35,11 @@ jobs: "${{ inputs.version }}", "${{ inputs.runner-type }}", ] + outputs: + vars: ${{ steps.vars.outputs.has_creds}} + vm_report: ${{ steps.set-multiple-vars.outputs.VM_REPORT }} + container_report: ${{ steps.set-multiple-vars.outputs.CONTAINER_REPORT }} + vm_serial_report: ${{ steps.set-multiple-vars.outputs.VM_SERIAL_REPORT }} steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: @@ -53,7 +58,7 @@ jobs: echo "has_creds=$has_creds" >> $GITHUB_OUTPUT - name: configure aws credentials uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2 - if: ${{ steps.vars.outputs.has_creds == true }} + if: ${{ steps.vars.outputs.has_creds == 'true' }} with: role-to-assume: ${{ secrets.ROLE }} role-session-name: credhelper-test @@ -82,5 +87,78 @@ jobs: run: | git status git clean -f -d - REGISTRY=${{ steps.vars.outputs.has_creds == true && env.REGISTRY || '' }} make ${{ inputs.test-command }} + if [[ "${{ github.event_name }}" == "push" ]] && [[ "${{ github.ref }}" == "refs/heads/main" ]] || [[ "${{ github.event_name }}" == "pull_request" ]] && [[ "${{ github.base_ref }}" == "main" ]]; then + REGISTRY=${{ steps.vars.outputs.has_creds == true && env.REGISTRY || '' }} make "${{ inputs.test-command }}-with-report" + else + REGISTRY=${{ steps.vars.outputs.has_creds == true && env.REGISTRY || '' }} make ${{ inputs.test-command }} + fi shell: zsh {0} + - name: Set artifacts name outputs + if: always() && (github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == 'pull_request' && github.base_ref == 'main') + id: set-multiple-vars + run: | + echo "VM_REPORT=${{ github.run_id }}-${{ github.run_attempt }}-e2e-vm-report.json" >> $GITHUB_OUTPUT + echo "CONTAINER_REPORT=${{ github.run_id }}-${{ github.run_attempt }}-e2e-container-report.json" >> $GITHUB_OUTPUT + echo "VM_SERIAL_REPORT=${{ github.run_id }}-${{ github.run_attempt }}-e2e-vm-serial-report.json" >> $GITHUB_OUTPUT + - name: Upload reports artifact + if: always() && (github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == 'pull_request' && github.base_ref == 'main') + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 + with: + name: macos-${{ inputs.version }}-${{ inputs.test-command }}-${{ inputs.arch }}-${{ github.run_id }}-${{ github.run_attempt }}-e2e-reports + path: ${{ github.workspace }}/reports/${{ github.run_id }}-${{ github.run_attempt }}-*.json + upload-macos-e2e-test-report: + needs: test + if: always() && ((github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == 'pull_request' && github.base_ref == 'main')) + runs-on: + [ + "self-hosted", + "macos", + "${{ inputs.arch }}", + "${{ inputs.version }}", + "${{ inputs.runner-type }}", + ] + steps: + - name: configure aws credentials + uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2 + if: ${{ needs.test.outputs.vars == 'true' }} + with: + role-to-assume: ${{ secrets.ROLE }} + role-session-name: credhelper-test + aws-region: ${{ secrets.REGION }} + - name: download artifacts + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.08 + with: + name: macos-${{ inputs.version }}-${{ inputs.test-command }}-${{ inputs.arch }}-${{ github.run_id }}-${{ github.run_attempt }}-e2e-reports + path: macos-${{ inputs.version }}-${{ inputs.test-command }}-${{ inputs.arch }}-${{ github.run_id }}-${{ github.run_attempt }}-e2e-reports + - name: Display structure of downloaded files + run: ls -R macos-${{ inputs.version }}-${{ inputs.test-command }}-${{ inputs.arch }}-${{ github.run_id }}-${{ github.run_attempt }}-e2e-reports + - name: Check and upload e2e tests reports to s3 bucket + if: always() + run: | + VM_REPORT="${{ github.workspace }}/macos-${{ inputs.version }}-${{ inputs.test-command }}-${{ inputs.arch }}-${{ github.run_id }}-${{ github.run_attempt }}-e2e-reports/${{ github.run_id }}-${{ github.run_attempt }}-e2e-vm-report.json" + CONTAINER_REPORT="${{ github.workspace }}/macos-${{ inputs.version }}-${{ inputs.test-command }}-${{ inputs.arch }}-${{ github.run_id }}-${{ github.run_attempt }}-e2e-reports/${{ github.run_id }}-${{ github.run_attempt }}-e2e-container-report.json" + VM_SERIAL_REPORT="${{ github.workspace }}/macos-${{ inputs.version }}-${{ inputs.test-command }}-${{ inputs.arch }}-${{ github.run_id }}-${{ github.run_attempt }}-e2e-reports/${{ github.run_id }}-${{ github.run_attempt }}-e2e-vm-serial-report.json" + + if [ -f "$VM_REPORT" ]; then + echo "VM report file exists. Uploading to S3..." + aws s3 cp "$VM_REPORT" "s3://finch-e2e-test-log-reports/macos-${{ inputs.arch }}/${{ needs.test.outputs.vm_report }}" + echo "VM report uploaded successfully." + else + echo "VM report file does not exist. Skipping upload." + fi + + if [ -f "$CONTAINER_REPORT" ]; then + echo "Container report file exists. Uploading to S3..." + aws s3 cp "$CONTAINER_REPORT" "s3://finch-e2e-test-log-reports/macos-${{ inputs.arch }}/${{ needs.test.outputs.container_report }}" + echo "Container report uploaded successfully." + else + echo "Container report file does not exist. Skipping upload." + fi + + if [ -f "$VM_SERIAL_REPORT" ]; then + echo "VM serial report file exists. Uploading to S3..." + aws s3 cp "$VM_SERIAL_REPORT" "s3://finch-e2e-test-log-reports/macos-${{ inputs.arch }}/${{ needs.test.outputs.vm_serial_report }}" + echo "VM serial report uploaded successfully." + else + echo "VM serial report file does not exist. Skipping upload." + fi \ No newline at end of file diff --git a/.github/workflows/e2e-windows.yaml b/.github/workflows/e2e-windows.yaml index 59c62cfb5..18f4796e6 100644 --- a/.github/workflows/e2e-windows.yaml +++ b/.github/workflows/e2e-windows.yaml @@ -32,6 +32,10 @@ jobs: "${{ inputs.arch }}", "${{ inputs.runner-type }}", ] + outputs: + vm_report: ${{ steps.set-multiple-vars.outputs.VM_REPORT }} + container_report: ${{ steps.set-multiple-vars.outputs.CONTAINER_REPORT }} + vm_serial_report: ${{ steps.set-multiple-vars.outputs.VM_SERIAL_REPORT }} steps: - name: Configure git CRLF settings run: | @@ -88,7 +92,29 @@ jobs: git status git clean -f -d - make ${{ inputs.test-command }} + + if ($env:GITHUB_EVENT_NAME -eq "push" -and $env:GITHUB_REF -eq "refs/heads/main") { + make "${{ inputs.test-command }}-with-report" + } + elseif ($env:GITHUB_EVENT_NAME -eq "pull_request" -and $env:GITHUB_BASE_REF -eq "main") { + make "${{ inputs.test-command }}-with-report" + } + else { + make ${{ inputs.test-command }} + } + - name: Set artifacts name outputs + if: always() && (github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == 'pull_request' && github.base_ref == 'main') + id: set-multiple-vars + run: | + "VM_REPORT=${{ github.run_id }}-${{ github.run_attempt }}-e2e-vm-report.json" >> $env:GITHUB_OUTPUT + "CONTAINER_REPORT=${{ github.run_id }}-${{ github.run_attempt }}-e2e-container-report.json" >> $env:GITHUB_OUTPUT + "VM_SERIAL_REPORT=${{ github.run_id }}-${{ github.run_attempt }}-e2e-vm-serial-report.json" >> $env:GITHUB_OUTPUT + - name: Upload reports artifact + if: always() && (github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == 'pull_request' && github.base_ref == 'main') + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 + with: + name: windows-${{ inputs.test-command }}-${{ inputs.arch }}-${{ github.run_id }}-${{ github.run_attempt }}-e2e-reports + path: ${{ github.workspace }}/reports/${{ github.run_id }}-${{ github.run_attempt }}-*.json - name: Remove Finch VM and Clean Up Previous Environment if: ${{ always() }} timeout-minutes: 2 @@ -97,3 +123,62 @@ jobs: make clean cd deps/finch-core && make clean exit 0 # Cleanup may set the exit code e.g. if a file doesn't exist; just ignore + upload-windows-e2e-test-report: + needs: test + timeout-minutes: 180 + runs-on: + [ + "self-hosted", + "windows", + "${{ inputs.arch }}", + "${{ inputs.runner-type }}", + ] + steps: + - name: Set output variables + id: vars + run: | + $has_creds="${{ (github.event_name == 'push' || github.repository == github.event.pull_request.head.repo.full_name) && github.actor != 'dependabot[bot]'}}" + echo "has_creds=$has_creds" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append + exit 0 # if $has_creds is false, powershell will exit with code 1 and this step will fail + - name: configure aws credentials + uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2 + if: env.has_creds == 'true' + with: + role-to-assume: ${{ secrets.ROLE }} + role-session-name: credhelper-test + aws-region: ${{ secrets.REGION }} + - name: download artifacts + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.08 + with: + name: windows-${{ inputs.test-command }}-${{ inputs.arch }}-${{ github.run_id }}-${{ github.run_attempt }}-e2e-reports + path: windows-${{ inputs.test-command }}-${{ inputs.arch }}-${{ github.run_id }}-${{ github.run_attempt }}-e2e-reports + - name: Check and upload e2e tests reports to s3 bucket + if: ${{ always() }} + run: | + $env:VM_REPORT = "${{ github.workspace }}\windows-${{ inputs.test-command }}-${{ inputs.arch }}-${{ github.run_id }}-${{ github.run_attempt }}-e2e-reports\${{ github.run_id }}-${{ github.run_attempt }}-e2e-vm-report.json" + $env:CONTAINER_REPORT = "${{ github.workspace }}\windows-${{ inputs.test-command }}-${{ inputs.arch }}-${{ github.run_id }}-${{ github.run_attempt }}-e2e-reports\${{ github.run_id }}-${{ github.run_attempt }}-e2e-container-report.json" + $env:VM_SERIAL_REPORT = "${{ github.workspace }}\windows-${{ inputs.test-command }}-${{ inputs.arch }}-${{ github.run_id }}-${{ github.run_attempt }}-e2e-reports\${{ github.run_id }}-${{ github.run_attempt }}-e2e-vm-serial-report.json" + + if (Test-Path $env:VM_REPORT) { + Write-Host "VM report file exists. Uploading to S3..." + aws s3 cp $env:VM_REPORT "s3://finch-e2e-test-log-reports/windows-${{ inputs.arch }}/${{ github.run_id }}-${{ github.run_attempt }}-e2e-vm-report.json" + Write-Host "VM report uploaded successfully." + } else { + Write-Host "VM report file does not exist. Skipping upload." + } + + if (Test-Path $env:CONTAINER_REPORT) { + Write-Host "Container report file exists. Uploading to S3..." + aws s3 cp $env:CONTAINER_REPORT "s3://finch-e2e-test-log-reports/windows-${{ inputs.arch }}/${{ github.run_id }}-${{ github.run_attempt }}-e2e-container-report.json" + Write-Host "Container report uploaded successfully." + } else { + Write-Host "Container report file does not exist. Skipping upload." + } + + if (Test-Path $env:VM_SERIAL_REPORT) { + Write-Host "VM serial report file exists. Uploading to S3..." + aws s3 cp $env:VM_SERIAL_REPORT "s3://finch-e2e-test-log-reports/windows-${{ inputs.arch }}/${{ github.run_id }}-${{ github.run_attempt }}-e2e-vm-serial-report.json" + Write-Host "VM serial report uploaded successfully." + } else { + Write-Host "VM serial report file does not exist. Skipping upload." + } \ No newline at end of file diff --git a/.github/workflows/test-pkg.yaml b/.github/workflows/test-pkg.yaml index 10cfbbf13..f8bcc51b5 100644 --- a/.github/workflows/test-pkg.yaml +++ b/.github/workflows/test-pkg.yaml @@ -10,7 +10,7 @@ on: required: true output_arch: type: string - required: true + required: true version: type: string required: true @@ -133,7 +133,7 @@ jobs: sudo rm -rf ./_output echo 'y' | sudo bash /Applications/Finch/uninstall.sh # Need to reinstall because there were errors on arm64 11.7 and arm64 12.6 hosts after running multiple instances tests, - # that caused the VM initialization failure in the e2e test. + # that caused the VM initialization failure in the e2e test. # Example workflow run https://github.com/runfinch/finch/actions/runs/4367457552/jobs/7638794529 sudo installer -pkg Finch-${{ inputs.tag }}-${{ inputs.output_arch }}.pkg -target / - name: Run VM e2e tests @@ -144,7 +144,7 @@ jobs: command: | git status git clean -f -d - INSTALLED=true make test-e2e-vm + INSTALLED=true make test-e2e-vm-with-report - name: Run container e2e tests uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e # v3.0.0 with: @@ -153,7 +153,7 @@ jobs: command: | git status git clean -f -d - INSTALLED=true make test-e2e-container + INSTALLED=true make test-e2e-container-with-report - name: Silently uninstall run: echo 'y' | sudo bash /Applications/Finch/uninstall.sh - name: Delete installer diff --git a/Makefile b/Makefile index 3ca10ea7f..0855b52f2 100644 --- a/Makefile +++ b/Makefile @@ -8,6 +8,12 @@ DEST := $(shell echo "$(DESTDIR)/$(PREFIX)" | sed 's:///*:/:g; s://*$$::') BINDIR ?= /usr/local/bin OUTDIR ?= $(CURDIR)/_output OS_OUTDIR ?= $(OUTDIR)/os +REPORT_DIR ?= $(CURDIR)/reports +RUN_ID ?= $(GITHUB_RUN_ID) +RUN_ATTEMPT ?= $(GITHUB_RUN_ATTEMPT) +REPORT_DIR ?= $(CURDIR)/reports +RUN_ID ?= $(GITHUB_RUN_ID) +RUN_ATTEMPT ?= $(GITHUB_RUN_ATTEMPT) OUTPUT_DIRECTORIES := $(OUTDIR) $(OS_OUTDIR) $(OUTPUT_DIRECTORIES): @@ -275,20 +281,39 @@ test-unit: # # Container tests and VM tests can be run in any order, but they must be run sequentially. # For more details, see the package-level comment of the e2e package. +# define e2e test command +COMMON_TEST_CMD_PREFIX := go test -ldflags $(LDFLAGS) -timeout 2h +COMMON_TEST_CMD_SUFFIX := -test.v -ginkgo.v -ginkgo.timeout=2h -ginkgo.flake-attempts=3 + .PHONY: test-e2e test-e2e: test-e2e-vm-serial test-e2e-container +.PHONY: test-e2e-with-report +test-e2e-with-report: test-e2e-vm-serial-with-report test-e2e-container-with-report test-e2e-vm-with-report + .PHONY: test-e2e-vm-serial -test-e2e-vm-serial: - go test -ldflags $(LDFLAGS) -timeout 2h ./e2e/vm -test.v -ginkgo.v -ginkgo.timeout=2h -ginkgo.flake-attempts=3 --installed="$(INSTALLED)" +test-e2e-vm-serial: + $(COMMON_TEST_CMD_PREFIX) ./e2e/vm $(COMMON_TEST_CMD_SUFFIX) --installed="$(INSTALLED)" .PHONY: test-e2e-container test-e2e-container: - go test -ldflags $(LDFLAGS) -timeout 2h ./e2e/container -test.v -ginkgo.v -ginkgo.timeout=2h -ginkgo.flake-attempts=3 --installed="$(INSTALLED)" + $(COMMON_TEST_CMD_PREFIX) ./e2e/container $(COMMON_TEST_CMD_SUFFIX) --installed="$(INSTALLED)" .PHONY: test-e2e-vm test-e2e-vm: - go test -ldflags $(LDFLAGS) -timeout 2h ./e2e/vm -test.v -ginkgo.v -ginkgo.timeout=2h -ginkgo.flake-attempts=3 --installed="$(INSTALLED)" --registry="$(REGISTRY)" + $(COMMON_TEST_CMD_PREFIX) ./e2e/vm $(COMMON_TEST_CMD_SUFFIX) --installed="$(INSTALLED)" --registry="$(REGISTRY)" + +.PHONY: test-e2e-vm-serial-with-report +test-e2e-vm-serial-with-report: + $(COMMON_TEST_CMD_PREFIX) ./e2e/vm $(COMMON_TEST_CMD_SUFFIX) -ginkgo.json-report=$(REPORT_DIR)/$(RUN_ID)-$(RUN_ATTEMPT)-e2e-vm-serial-report.json --installed="$(INSTALLED)" + +.PHONY: test-e2e-container-with-report +test-e2e-container-with-report: + $(COMMON_TEST_CMD_PREFIX) ./e2e/container $(COMMON_TEST_CMD_SUFFIX) -ginkgo.json-report=$(REPORT_DIR)/$(RUN_ID)-$(RUN_ATTEMPT)-e2e-container-report.json --installed="$(INSTALLED)" + +.PHONY: test-e2e-vm-with-report +test-e2e-vm-with-report: + $(COMMON_TEST_CMD_PREFIX) ./e2e/vm $(COMMON_TEST_CMD_SUFFIX) -ginkgo.json-report=$(REPORT_DIR)/$(RUN_ID)-$(RUN_ATTEMPT)-e2e-vm-report.json --installed="$(INSTALLED)" --registry="$(REGISTRY)" .PHONY: test-benchmark test-benchmark: