ci.yaml

name: CI

on:
  pull_request: {}
  push:
    branches:
      - main

concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

permissions:
  contents: write

jobs:
  git-sha:
    name: Git SHA
    runs-on: ubuntu-latest
    outputs:
      git_sha: ${{ steps.git_sha.outputs.git_sha }}
    steps:
      - uses: actions/checkout@v4
      - uses: ./.github/actions/git-sha
        id: git_sha

  sanitize:
    name: Sanitize
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Setup Go
        uses: actions/setup-go@v5
        with:
          go-version-file: go.mod
          cache-dependency-path: "**/*.sum"
      - name: Go vet
        run: |
          make vet
      - name: Lint
        uses: golangci/golangci-lint-action@v6
        with:
          version: v1.60.3

  test:
    name: Unit tests
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Setup Go
        uses: actions/setup-go@v5
        with:
          go-version-file: go.mod
          cache-dependency-path: "**/*.sum"
      - name: Unit tests
        run: |
          make unit-tests

  check-operator-crds:
    name: Check Operator CRDs
    runs-on: ubuntu-latest
    steps:
    - name: Checkout
      uses: actions/checkout@v4
    - name: Make Manifests
      run: make -C operator manifests
    - name: Check CRDs
      run: |
        git diff --exit-code --name-only
        if [ $? -eq 0 ]; then
          echo "CRDs are up to date"
        else
          echo "CRDs are out of date"
          exit 1
        fi

  build-operator:
    name: Build Operator
    runs-on: ubuntu-latest
    needs: [git-sha]
    outputs:
      operator-version: ${{ steps.operator-version.outputs.version }}
      operator-image: ${{ steps.operator-image.outputs.image }}
      operator-chart: ${{ steps.operator-chart.outputs.chart }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0 # fetch all history so that we can get the previous tag

      - name: Cache Melange
        uses: actions/cache@v4
        with:
          path: |
            operator/build/.melange-cache
          key: melange-cache

      - name: Setup Melange
        uses: chainguard-dev/actions/setup-melange@main
      
      - name: Output operator version
        id: operator-version
        run: |
          echo "version=$(git describe --tags --dirty --match='[0-9]*.[0-9]*.[0-9]*')" >> $GITHUB_OUTPUT

      - name: Build and push operator image
        id: operator-image
        run: |
          make -C operator apko build-ttl.sh
          echo "image=$(cat operator/build/image)" >> $GITHUB_OUTPUT

      - name: Build and push operator chart
        id: operator-chart
        run: |
          make -C operator build-chart-ttl.sh
          echo "chart=$(cat operator/build/chart)" >> $GITHUB_OUTPUT

  build-images:
    name: Build Images
    outputs:
      local-artifact-mirror: ${{ steps.local-artifact-mirror.outputs.image }}
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0 # fetch all history so that we can get the previous tag

      - name: Cache Melange
        uses: actions/cache@v4
        with:
          path: |
            local-artifact-mirror/build/.melange-cache
          key: melange-cache

      - name: Setup Melange
        uses: chainguard-dev/actions/setup-melange@main

      - name: Build and push local-artifact-mirror image
        id: local-artifact-mirror
        run: |
          make -C local-artifact-mirror apko build-ttl.sh
          echo "image=$(cat local-artifact-mirror/build/image)" >> $GITHUB_OUTPUT

  buildtools:
    name: Build Buildtools
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Setup Go
        uses: actions/setup-go@v5
        with:
          go-version-file: go.mod
          cache-dependency-path: "**/*.sum"
      - name: Compile buildtools
        run: |
          make buildtools
      - name: Upload buildtools artifact
        uses: actions/upload-artifact@v4
        with:
          name: buildtools
          path: output/bin/buildtools

  build:
    name: Build
    runs-on: ubuntu-latest
    needs: [git-sha, buildtools, build-images, build-operator]
    outputs:
      k0s_version: ${{ steps.export.outputs.k0s_version }}
      k0s_version_previous: ${{ steps.export.outputs.k0s_version_previous }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Setup Go
        uses: actions/setup-go@v5
        with:
          go-version-file: go.mod
          cache-dependency-path: "**/*.sum"

      - name: Download buildtools artifact
        uses: actions/download-artifact@v4
        with:
          name: buildtools
          path: output/bin

      - name: Update embedded-cluster-operator metadata.yaml
        env:
          IMAGES_REGISTRY_SERVER: ttl.sh
          OPERATOR_CHART: ${{ needs.build-operator.outputs.operator-chart }}
          OPERATOR_IMAGE: ${{ needs.build-operator.outputs.operator-image }}
        run: |
          ./scripts/ci-update-operator-metadata.sh

      - name: Build Linux AMD64 and Output Metadata
        run: |
          export SHORT_SHA=dev-${{ needs.git-sha.outputs.git_sha }}
          export LOCAL_ARTIFACT_MIRROR_IMAGE=proxy.replicated.com/anonymous/${{ needs.build-images.outputs.local-artifact-mirror }}
          export S3_BUCKET_PUBLIC_URL=https://dev-embedded-cluster-bin.s3.amazonaws.com

          # build previous k0s
          export EC_VERSION="${SHORT_SHA}-previous-k0s"
          export K0S_VERSION=$(make print-PREVIOUS_K0S_VERSION)
          export KOTS_VERSION=$(make print-KOTS_VERSION)
          export OPERATOR_VERSION=${{ needs.build-operator.outputs.operator-version }}
          ./scripts/ci-build.sh
          mv embedded-cluster-linux-amd64.tgz embedded-cluster-linux-amd64-previous-k0s.tgz
          mv metadata.json metadata-previous-k0s.json

          # build upgrade
          export EC_VERSION="${SHORT_SHA}-upgrade"
          export K0S_VERSION=$(make print-K0S_VERSION)
          export KOTS_VERSION=$(make print-KOTS_VERSION)
          export OPERATOR_VERSION=${{ needs.build-operator.outputs.operator-version }}
          ./scripts/ci-build.sh
          mv embedded-cluster-linux-amd64.tgz embedded-cluster-linux-amd64-upgrade.tgz
          mv metadata.json metadata-upgrade.json

          # build current
          export EC_VERSION="${SHORT_SHA}"
          export K0S_VERSION=$(make print-K0S_VERSION)
          export KOTS_VERSION=$(make print-KOTS_VERSION)
          export OPERATOR_VERSION=${{ needs.build-operator.outputs.operator-version }}
          ./scripts/ci-build.sh

      - name: Upload binary
        uses: actions/upload-artifact@v4
        with:
          name: embedded-binary
          path: |
            output/bin/embedded-cluster

      - name: Build CI binary
        run: |
          export SHORT_SHA=dev-${{ needs.git-sha.outputs.git_sha }}
          export EC_VERSION="${SHORT_SHA}"
          export APP_VERSION="appver-${SHORT_SHA}"

          export LOCAL_ARTIFACT_MIRROR_IMAGE=proxy.replicated.com/anonymous/${{ needs.build-images.outputs.local-artifact-mirror }}
          export OPERATOR_IMAGE=proxy.replicated.com/anonymous/${{ needs.build-operator.outputs.operator-image }}
          echo "# channel release object" > e2e/kots-release-install/release.yaml
          echo 'channelID: "2cHXb1RCttzpR0xvnNWyaZCgDBP"' >> e2e/kots-release-install/release.yaml
          echo 'channelSlug: "ci"' >> e2e/kots-release-install/release.yaml
          echo 'appSlug: "embedded-cluster-smoke-test-staging-app"' >> e2e/kots-release-install/release.yaml
          echo "versionLabel: \"${APP_VERSION}\"" >> e2e/kots-release-install/release.yaml
          cat e2e/kots-release-install/release.yaml

          # ensure that the cluster config embedded in the CI binaries is correct
          sed -i "s/__version_string__/${EC_VERSION}/g" e2e/kots-release-install/cluster-config.yaml
          sed -i "s/__version_string__/${EC_VERSION}-upgrade/g" e2e/kots-release-upgrade/cluster-config.yaml

          cp output/bin/embedded-cluster output/bin/embedded-cluster-original
          make -B embedded-release K0S_VERSION=$(make print-PREVIOUS_K0S_VERSION) VERSION="${EC_VERSION}-previous-k0s"
          mv output/bin/embedded-cluster output/bin/embedded-cluster-previous-k0s
          make -B embedded-release VERSION="${EC_VERSION}" # this is done after the metadata.json is generated so as to not include additional charts

      - name: Cache files for integration test
        env:
          S3_BUCKET: "dev-embedded-cluster-bin"
          AWS_ACCESS_KEY_ID: ${{ secrets.DEV_EMBEDDED_CLUSTER_UPLOAD_IAM_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_EMBEDDED_CLUSTER_UPLOAD_IAM_SECRET }}
          AWS_REGION: "us-east-1"
        run: |
          export SHORT_SHA=dev-${{ needs.git-sha.outputs.git_sha }}
          export EC_VERSION="${SHORT_SHA}"
          export OPERATOR_IMAGE=proxy.replicated.com/anonymous/${{ needs.build-operator.outputs.operator-image }}
          export OPERATOR_VERSION=${{ needs.build-operator.outputs.operator-version }}

          ./scripts/cache-files.sh
          ./scripts/create-upgrade-release.sh
          ./scripts/create-previous-k0s-release.sh

      - name: Upload release
        uses: actions/upload-artifact@v4
        with:
          name: embedded-release
          path: |
            output/bin/embedded-cluster
            output/bin/embedded-cluster-original
            output/bin/embedded-cluster-previous-k0s
            output/bin/embedded-cluster-release-builder

      - name: Export K0s Version
        id: export
        run: |
          K0S_VERSION="$(make print-K0S_VERSION)"
          echo "K0S_VERSION=\"$K0S_VERSION\""
          echo "k0s_version=$K0S_VERSION" >> "$GITHUB_OUTPUT"
          PREVIOUS_K0S_VERSION="$(make print-PREVIOUS_K0S_VERSION)"
          echo "PREVIOUS_K0S_VERSION=\"$PREVIOUS_K0S_VERSION\""
          echo "k0s_version_previous=$PREVIOUS_K0S_VERSION" >> "$GITHUB_OUTPUT"

  check-images:
    name: Check Images
    runs-on: ubuntu-latest
    needs: [buildtools, build]
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Download buildtools artifact
        uses: actions/download-artifact@v4
        with:
          name: buildtools
          path: output/bin
      - name: Download embedded-cluster artifact
        uses: actions/download-artifact@v4
        with:
          name: embedded-binary
          path: output/bin
      - name: Check for missing images
        run: |
          chmod +x ./output/bin/buildtools
          chmod +x ./output/bin/embedded-cluster
          ./output/bin/embedded-cluster version metadata > version-metadata.json
          ./output/bin/embedded-cluster version list-images > expected.txt
          printf "Expected images:\n$(cat expected.txt)\n"
          ./output/bin/buildtools metadata extract-helm-chart-images --metadata-path version-metadata.json > images.txt
          printf "Found images:\n$(cat images.txt)\n"
          missing_images=0
          while read img; do
            grep -q "$img" expected.txt || { echo "Missing image: $img" && missing_images=$((missing_images+1)) ; }
          done <images.txt
          if [ $missing_images -gt 0 ]; then
            echo "Found $missing_images missing images"
            exit 1
          fi

  release-app:
    name: Create App Releases
    runs-on: ubuntu-latest
    permissions:
      pull-requests: write
    needs:
      - git-sha
      - build
      - build-images
      - build-operator
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Install Replicated CLI
        run: |
          curl --retry 5 --retry-all-errors -fs https://api.github.com/repos/replicatedhq/replicated/releases/latest \
          | grep "browser_download_url.*linux_amd64.tar.gz" \
          | cut -d : -f 2,3 \
          | tr -d \" \
          | wget -O replicated.tar.gz -qi -
          tar xf replicated.tar.gz replicated && rm replicated.tar.gz
          mv replicated /usr/local/bin/replicated
      - name: Create CI Releases
        env:
          REPLICATED_APP: "embedded-cluster-smoke-test-staging-app"
          REPLICATED_API_TOKEN: ${{ secrets.STAGING_REPLICATED_API_TOKEN }}
          REPLICATED_API_ORIGIN: "https://api.staging.replicated.com/vendor"
        run: |
          export SHORT_SHA=dev-${{ needs.git-sha.outputs.git_sha }}
          export EC_VERSION="${SHORT_SHA}"
          export APP_VERSION="appver-${SHORT_SHA}"

          sed -i "s/__version_string__/${EC_VERSION}/g" e2e/kots-release-install/cluster-config.yaml
          sed -i "s/__version_string__/${EC_VERSION}-upgrade/g" e2e/kots-release-upgrade/cluster-config.yaml

          # re-promote a release containing an old version of embedded-cluster to test upgrades
          replicated release promote 807 2cHXb1RCttzpR0xvnNWyaZCgDBP --version "${APP_VERSION}-pre-minio-removal"

          # install the previous k0s version to ensure an upgrade occurs
          sed -i "s/${EC_VERSION}/${EC_VERSION}-previous-k0s/g" e2e/kots-release-install/cluster-config.yaml
          replicated release create --yaml-dir e2e/kots-release-install --promote CI --version "${APP_VERSION}-previous-k0s"
          # return the cluster config to the current version
          sed -i "s/${EC_VERSION}-previous-k0s/${EC_VERSION}/g" e2e/kots-release-install/cluster-config.yaml

          replicated release create --yaml-dir e2e/kots-release-install --promote CI --version "${APP_VERSION}"
          replicated release create --yaml-dir e2e/kots-release-install --promote CI --version "${APP_VERSION}-noop"
          replicated release create --yaml-dir e2e/kots-release-upgrade --promote CI --version "${APP_VERSION}-upgrade"

      - name: Create Airgap Release
        env:
          REPLICATED_APP: "embedded-cluster-smoke-test-staging-app"
          REPLICATED_API_TOKEN: ${{ secrets.STAGING_REPLICATED_API_TOKEN }}
          REPLICATED_API_ORIGIN: "https://api.staging.replicated.com/vendor"
        run: |
          export SHORT_SHA=dev-${{ needs.git-sha.outputs.git_sha }}
          export EC_VERSION="${SHORT_SHA}"
          export APP_VERSION="appver-${SHORT_SHA}"

          # airgap tests install the previous k0s version to ensure an upgrade occurs
          sed -i "s/${EC_VERSION}/${EC_VERSION}-previous-k0s/g" e2e/kots-release-install/cluster-config.yaml
          replicated release create --yaml-dir e2e/kots-release-install --promote CI-airgap --version "${APP_VERSION}-previous-k0s"
          # return the cluster config to the current version
          sed -i "s/${EC_VERSION}-previous-k0s/${EC_VERSION}/g" e2e/kots-release-install/cluster-config.yaml

          replicated release create --yaml-dir e2e/kots-release-install --promote CI-airgap --version "${APP_VERSION}"
          replicated release create --yaml-dir e2e/kots-release-upgrade --promote CI-airgap --version "${APP_VERSION}-upgrade"

      - name: Create download link message text
        if: github.event_name == 'pull_request'
        run: |
          export SHORT_SHA=dev-${{ needs.git-sha.outputs.git_sha }}
          export EC_VERSION="${SHORT_SHA}"
          export APP_VERSION="appver-${SHORT_SHA}"
          
          echo "This PR has been released (on staging) and is available for download with a embedded-cluster-smoke-test-staging-app [license ID](https://vendor.staging.replicated.com/apps/embedded-cluster-smoke-test-staging-app/customers?sort=name-asc)." > download-link.txt
          echo "" >> download-link.txt
          echo "Online Installer:" >> download-link.txt
          echo "\`\`\`" >> download-link.txt
          echo "curl \"https://staging.replicated.app/embedded/embedded-cluster-smoke-test-staging-app/ci/${APP_VERSION}\" -H \"Authorization: \$EC_SMOKE_TEST_LICENSE_ID\" -o embedded-cluster-smoke-test-staging-app-ci.tgz" >> download-link.txt
          echo "\`\`\`" >> download-link.txt
          echo "Airgap Installer (may take a few minutes before the airgap bundle is built):" >> download-link.txt
          echo "\`\`\`" >> download-link.txt
          echo "curl \"https://staging.replicated.app/embedded/embedded-cluster-smoke-test-staging-app/ci-airgap/${APP_VERSION}?airgap=true\" -H \"Authorization: \$EC_SMOKE_TEST_LICENSE_ID\" -o embedded-cluster-smoke-test-staging-app-ci.tgz" >> download-link.txt
          echo "\`\`\`" >> download-link.txt
          echo "Happy debugging!" >> download-link.txt
          cat download-link.txt

      - name: comment download link
        if: github.event_name == 'pull_request'
        uses: mshick/add-pr-comment@v2
        with:
          message-path: download-link.txt

  # e2e-docker runs the e2e tests inside a docker container rather than a full VM
  e2e-docker:
    runs-on: ubuntu-latest
    needs:
      - build
    strategy:
      fail-fast: false
      matrix:
        test:
          - TestPreflights
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Download binary
        uses: actions/download-artifact@v4
        with:
          name: embedded-release
          path: output/bin
      - name: Setup Go
        uses: actions/setup-go@v5
        with:
          go-version-file: go.mod
          cache-dependency-path: "**/*.sum"
      - name: Write license file
        run: |
          echo "${{ secrets.STAGING_EMBEDDED_CLUSTER_LICENSE }}" | base64 --decode > e2e/license.yaml
      - name: Run test
        env:
          LICENSE_PATH: license.yaml
        run: |
          make e2e-test TEST_NAME=${{ matrix.test }}

  e2e:
    runs-on: ${{ matrix.runner || 'ubuntu-latest' }}
    needs:
      - build
      - build-images
      - build-operator
      - release-app
    strategy:
      fail-fast: false
      matrix:
        test:
          - TestSingleNodeInstallation
          - TestSingleNodeInstallationAlmaLinux8
          - TestSingleNodeInstallationDebian11
          - TestSingleNodeInstallationDebian12
          - TestSingleNodeInstallationCentos9Stream
          - TestVersion
          - TestHostPreflightCustomSpec
          - TestHostPreflightInBuiltSpec
          - TestUnsupportedOverrides
          - TestMultiNodeInstallation
          - TestMultiNodeReset
          - TestCommandsRequireSudo
          - TestInstallWithoutEmbed
          - TestInstallFromReplicatedApp
          - TestUpgradeFromReplicatedApp
          - TestUpgradeEC18FromReplicatedApp
          - TestResetAndReinstall
          - TestResetAndReinstallAirgap
          - TestCollectSupportBundle
          - TestOldVersionUpgrade
          - TestMaterialize
          - TestLocalArtifactMirror
          - TestSingleNodeAirgapUpgrade
          - TestSingleNodeAirgapUpgradeFromEC18
          - TestSingleNodeAirgapUpgradeCustomCIDR
          - TestInstallSnapshotFromReplicatedApp
          - TestMultiNodeAirgapUpgrade
          - TestSingleNodeDisasterRecovery
          - TestSingleNodeDisasterRecoveryWithProxy
          - TestSingleNodeResumeDisasterRecovery
          - TestProxiedEnvironment
          - TestMultiNodeHAInstallation
          - TestMultiNodeHADisasterRecovery
          - TestCustomCIDR
          - TestProxiedCustomCIDR
          - TestSingleNodeInstallationNoopUpgrade
        include:
          - test: TestMultiNodeAirgapUpgrade
            runner: embedded-cluster
          - test: TestMultiNodeAirgapUpgradeSameK0s
            runner: embedded-cluster
          - test: TestSingleNodeAirgapDisasterRecovery
            runner: embedded-cluster
          - test: TestMultiNodeAirgapHAInstallation
            runner: embedded-cluster
          - test: TestMultiNodeAirgapHADisasterRecovery
            runner: embedded-cluster
          - test: TestFiveNodesAirgapUpgrade
            runner: embedded-cluster
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: download binary
        uses: actions/download-artifact@v4
        with:
          name: embedded-release
          path: output/bin

      - uses: ./.github/actions/e2e
        with:
          test-name: '${{ matrix.test }}'
          is-large-runner: ${{ matrix.runner == 'embedded-cluster' }}
          airgap-license-id: ${{ secrets.STAGING_EMBEDDED_CLUSTER_AIRGAP_LICENSE_ID }}
          snapshot-license-id: ${{ secrets.STAGING_EMBEDDED_CLUSTER_SNAPSHOT_LICENSE_ID }}
          snapshot-license: ${{ secrets.STAGING_EMBEDDED_CLUSTER_SNAPSHOT_LICENSE }}
          airgap-snapshot-license-id: ${{ secrets.STAGING_EMBEDDED_CLUSTER_AIRGAP_SNAPSHOT_LICENSE_ID }}
          license-id: ${{ secrets.STAGING_EMBEDDED_CLUSTER_LICENSE_ID }}
          license: ${{ secrets.STAGING_EMBEDDED_CLUSTER_LICENSE }}
          dr-aws-access-key-id: ${{ secrets.TESTIM_AWS_ACCESS_KEY_ID }}
          dr-aws-secret-access-key: ${{ secrets.TESTIM_AWS_SECRET_ACCESS_KEY }}
          k0s-version: ${{ needs.build.outputs.k0s_version }}
          k0s-version-previous: ${{ needs.build.outputs.k0s_version_previous }}

  # this job will validate that all the tests passed
  # it is used for the github branch protection rule
  validate-success:
    runs-on: ubuntu-20.04
    needs:
      - e2e
      - e2e-docker
      - sanitize
      - test
      - check-images
      - check-operator-crds
    if: always()
    steps:
      # https://docs.github.com/en/actions/learn-github-actions/contexts#needs-context
      - name: fail if e2e job was not successful
        if: needs.e2e.result != 'success'
        run: exit 1
      - name: fail if sanitize job was not successful
        if: needs.sanitize.result != 'success'
        run: exit 1
      - name: fail if tests job was not successful
        if: needs.test.result != 'success'
        run: exit 1
      - name: succeed if everything else passed
        run: echo "Validation succeeded"