Debug e2e workflow execution

percona · Jan 24, 2025 · 6ef43da · 6ef43da
1 parent 85ae6bc
commit 6ef43da
Showing 1 changed file with 192 additions and 0 deletions.
diff --git a/.github/workflows/temp-rm-me.yml b/.github/workflows/temp-rm-me.yml
@@ -0,0 +1,192 @@
+name: CI
+on:
+  workflow_dispatch:
+    inputs:
+      debug:
+        type: choice
+        description: "Enable Tmate session for debugging."
+        options:
+        - Disabled
+        - Enabled
+        required: false
+
+jobs:
+  eks-e2e-tests:
+    name: E2E tests (EKS)
+    timeout-minutes: 80
+    concurrency: eks-e2e-tests
+    strategy:
+      fail-fast: true
+    runs-on: ubuntu-latest
+    env:
+      TOOLS_PATH: "/opt/tools/bin"
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v4
+
+      - name: Set up Go release
+        uses: actions/setup-go@v5
+        with:
+          go-version: '1.22'
+          check-latest: true
+
+      - name: Build operator
+        run:
+          make build
+
+      - name: Configure AWS
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-access-key-id: ${{ secrets.EVEREST_EKS_E2E_IAM_USER_ACCESS_KEY }}
+          aws-secret-access-key: ${{ secrets.EVEREST_EKS_E2E_IAM_USER_ACCESS_KEY_SECRET }}
+          aws-region: us-east-2
+
+      - name: Configure env
+        run: |
+          mkdir -p $TOOLS_PATH
+          echo $TOOLS_PATH >> $GITHUB_PATH
+
+          aws eks update-kubeconfig --name everest-e2e-dev-eks-cluster
+
+      - name: Install tools
+        run: |
+          # Kubectl
+          curl -LO https://dl.k8s.io/release/v$KUBECTL_VERSION/bin/linux/amd64/kubectl
+          curl -LO "https://dl.k8s.io/v$KUBECTL_VERSION/bin/linux/amd64/kubectl.sha256"
+          echo "$(cat kubectl.sha256)  kubectl" | sha256sum --check
+          chmod +x kubectl
+          cp kubectl "$TOOLS_PATH"
+          rm kubectl
+          # Kuttl
+          curl -Lo ./kubectl-kuttl https://github.com/kudobuilder/kuttl/releases/download/v0.15.0/kubectl-kuttl_0.15.0_linux_x86_64
+          chmod +x ./kubectl-kuttl
+          mv ./kubectl-kuttl "$TOOLS_PATH"
+          # Install EKSCTL
+          ARCH=amd64
+          PLATFORM=$(uname -s)_$ARCH
+          curl -sLO "https://github.com/eksctl-io/eksctl/releases/latest/download/eksctl_$PLATFORM.tar.gz"
+          tar -xzf eksctl_$PLATFORM.tar.gz -C "$TOOLS_PATH"
+          chmod +x "$TOOLS_PATH"/eksctl
+        env:
+          KUBECTL_VERSION: 1.27.9
+
+      - name: Scale up cluster
+        run: |
+          eksctl scale nodegroup --cluster="$CLUSTER_NAME" --nodes=10 "$NODEPOOL_NAME" --wait
+        env:
+          CLUSTER_NAME: "everest-e2e-dev-eks-cluster"
+          NODEPOOL_NAME: "everest-e2e-dev-eks-workers"
+
+      - name: Run upgrade tests
+        if: contains(github.event.pull_request.labels.*.name, 'test/e2e') || contains(github.event.pull_request.labels.*.name, 'test/e2e-upgrade')
+        run: |
+          kubectl kuttl test --config ./tests/e2e/kuttl-upgrade.yaml
+        env:
+          PXC_OPERATOR_VERSION: 1.15.0
+          PSMDB_OPERATOR_VERSION: 1.18.0
+          PG_OPERATOR_VERSION: 2.5.0
+          PREVIOUS_PG_OPERATOR_VERSION: 2.4.1
+          PREVIOUS_PXC_OPERATOR_VERSION: 1.14.0
+          PREVIOUS_PSMDB_OPERATOR_VERSION: 1.17.0
+          PERCONA_VERSION_SERVICE_URL: https://check-dev.percona.com/versions/v1
+
+      - name: Clean up custom resources
+        if: always()
+        continue-on-error: true
+        # If this does not finish in 10 minutes, it's stuck with some resource having a finalizer.
+        timeout-minutes: 10
+        run: |
+          namespaces=$(kubectl get pxc -A -o jsonpath='{.items[*].metadata.namespace}')
+          for namespace in ${namespaces[@]}
+          do
+            kubectl -n $namespace get pxc -o name | awk -F '/' {'print $2'} | xargs --no-run-if-empty kubectl patch pxc -n $namespace -p '{"metadata":{"finalizers":null}}' --type merge
+          done
+
+          namespaces=$(kubectl get psmdb -A -o jsonpath='{.items[*].metadata.namespace}')
+          for namespace in ${namespaces[@]}
+          do
+            kubectl -n $namespace get psmdb -o name | awk -F '/' {'print $2'} | xargs --no-run-if-empty kubectl patch psmdb -n $namespace -p '{"metadata":{"finalizers":null}}' --type merge
+          done
+
+          namespaces=$(kubectl get pg -A -o jsonpath='{.items[*].metadata.namespace}')
+          for namespace in ${namespaces[@]}
+          do
+            kubectl -n $namespace get pg -o name | awk -F '/' {'print $2'} | xargs --no-run-if-empty kubectl patch pg -n $namespace -p '{"metadata":{"finalizers":null}}' --type merge
+          done
+
+          namespaces=$(kubectl get db -A -o jsonpath='{.items[*].metadata.namespace}')
+          for namespace in ${namespaces[@]}
+          do
+            kubectl -n $namespace get db -o name | awk -F '/' {'print $2'} | xargs --no-run-if-empty kubectl patch db -n $namespace -p '{"metadata":{"finalizers":null}}' --type merge
+          done
+
+          kubectl delete db --all-namespaces --all --cascade=foreground
+          kubectl delete pvc --all-namespaces --all
+          kubectl delete backupstorage --all-namespaces --all
+          kubectl get ns -o name | grep kuttl  | awk -F '/' {'print $2'} | xargs --no-run-if-empty kubectl delete ns
+
+
+      - name: Run core tests
+        if: contains(github.event.pull_request.labels.*.name, 'test/e2e') || contains(github.event.pull_request.labels.*.name, 'test/e2e-core')
+        run: |
+          kubectl kuttl test --config ./tests/e2e/kuttl-core.yaml
+        env:
+          PXC_OPERATOR_VERSION: 1.15.0
+          PSMDB_OPERATOR_VERSION: 1.18.0
+          PG_OPERATOR_VERSION: 2.5.0
+          PERCONA_VERSION_SERVICE_URL: https://check-dev.percona.com/versions/v1
+
+      - name: Clean up custom resources
+        if: always()
+        run: |
+          namespaces=$(kubectl get pxc -A -o jsonpath='{.items[*].metadata.namespace}')
+          for namespace in ${namespaces[@]}
+          do
+            kubectl -n $namespace get pxc -o name | awk -F '/' {'print $2'} | xargs --no-run-if-empty kubectl patch pxc -n $namespace -p '{"metadata":{"finalizers":null}}' --type merge
+          done
+
+          namespaces=$(kubectl get psmdb -A -o jsonpath='{.items[*].metadata.namespace}')
+          for namespace in ${namespaces[@]}
+          do
+            kubectl -n $namespace get psmdb -o name | awk -F '/' {'print $2'} | xargs --no-run-if-empty kubectl patch psmdb -n $namespace -p '{"metadata":{"finalizers":null}}' --type merge
+          done
+
+          namespaces=$(kubectl get pg -A -o jsonpath='{.items[*].metadata.namespace}')
+          for namespace in ${namespaces[@]}
+          do
+            kubectl -n $namespace get pg -o name | awk -F '/' {'print $2'} | xargs --no-run-if-empty kubectl patch pg -n $namespace -p '{"metadata":{"finalizers":null}}' --type merge
+          done
+
+          namespaces=$(kubectl get db -A -o jsonpath='{.items[*].metadata.namespace}')
+          for namespace in ${namespaces[@]}
+          do
+            kubectl -n $namespace get db -o name | awk -F '/' {'print $2'} | xargs --no-run-if-empty kubectl patch db -n $namespace -p '{"metadata":{"finalizers":null}}' --type merge
+          done
+
+          kubectl delete db --all-namespaces --all --cascade=foreground
+          kubectl delete pvc --all-namespaces --all
+          kubectl delete backupstorage --all-namespaces --all
+          kubectl get ns -o name | grep kuttl  | awk -F '/' {'print $2'} | xargs --no-run-if-empty kubectl delete ns
+          kubectl delete ns operators olm --ignore-not-found=true --wait=false
+          sleep 10
+          kubectl delete apiservice v1.packages.operators.coreos.com --ignore-not-found=true
+          kubectl get crd -o name | grep .coreos.com$ | awk -F '/' {'print $2'} | xargs --no-run-if-empty kubectl delete crd
+
+      - name: Scale down cluster
+        if: always()
+        run: |
+          eksctl scale nodegroup --cluster="$CLUSTER_NAME" --nodes=0 "$NODEPOOL_NAME"
+        env:
+          CLUSTER_NAME: "everest-e2e-dev-eks-cluster"
+          NODEPOOL_NAME: "everest-e2e-dev-eks-workers"
+
+      - name: Clean up resources and CRDs
+        if: always()
+        run: |
+          kubectl get crd -o name | grep .percona.com$ | awk -F '/' {'print $2'} | xargs --no-run-if-empty kubectl delete crd
+          kubectl delete crd postgresclusters.postgres-operator.crunchydata.com
+
+      # Enable Tmate debug ssh session on demand
+      - name: "Setup tmate session"
+        uses: "percona/gh-action-action-tmate@v3"
+        if: ${{ always() && github.event.inputs.debug == 'Enabled' }}