diff --git a/.github/actions/e2e/action.yml b/.github/actions/e2e/action.yml index 5c865cf9f..ec8d5d668 100644 --- a/.github/actions/e2e/action.yml +++ b/.github/actions/e2e/action.yml @@ -31,6 +31,9 @@ inputs: dr-aws-secret-access-key: description: 'Disaster Recovery AWS Secret Access Key' required: true + k0s-version: + description: 'k0s version to expect after upgrades in e2e tests' + required: true runs: using: composite @@ -104,6 +107,7 @@ runs: export DR_AWS_S3_PREFIX_AIRGAP=${{ inputs.test-name }}-${{ github.run_id }}-${{ github.run_attempt }}-airgap export DR_AWS_ACCESS_KEY_ID=${{ inputs.dr-aws-access-key-id }} export DR_AWS_SECRET_ACCESS_KEY=${{ inputs.dr-aws-secret-access-key }} + export K0S_VERSION=${{ inputs.k0s-version }} make e2e-test TEST_NAME=${{ inputs.test-name }} - name: Upload Host Support Bundle uses: actions/upload-artifact@v4 diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b440ab14b..0d833bb68 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -176,6 +176,10 @@ jobs: output/bin/embedded-cluster-original output/bin/embedded-cluster-previous-k0s output/bin/embedded-cluster-release-builder + - name: Export K0s Version + run: | + make print-K0S_VERSION + echo "k0s_version=$(make print-K0S_VERSION)" >> "$GITHUB_OUTPUT" check-images: name: Check Images @@ -248,6 +252,12 @@ jobs: # re-promote a release containing an old version of embedded-cluster to test upgrades replicated release promote 807 2cHXb1RCttzpR0xvnNWyaZCgDBP --version "appver-${SHORT_SHA}-pre-minio-removal" + # install the previous k0s version to ensure an upgrade occurs + sed -i "s/${SHORT_SHA}/${SHORT_SHA}-previous-k0s/g" e2e/kots-release-install/cluster-config.yaml + replicated release create --yaml-dir e2e/kots-release-install --promote CI --version "appver-${SHORT_SHA}-previous-k0s" + # return the cluster config to the current version + sed -i "s/${SHORT_SHA}-previous-k0s/${SHORT_SHA}/g" e2e/kots-release-install/cluster-config.yaml + replicated release create --yaml-dir e2e/kots-release-install --promote CI --version "appver-${SHORT_SHA}" replicated release create --yaml-dir e2e/kots-release-install --promote CI --version "appver-${SHORT_SHA}-noop" replicated release create --yaml-dir e2e/kots-release-upgrade --promote CI --version "appver-${SHORT_SHA}-upgrade" @@ -260,12 +270,14 @@ jobs: run: | export SHORT_SHA=dev-${GITHUB_SHA::7} echo "${SHORT_SHA}" - replicated release create --yaml-dir e2e/kots-release-install --promote CI-airgap --version "appver-${SHORT_SHA}" # airgap tests install the previous k0s version to ensure an upgrade occurs sed -i "s/${SHORT_SHA}/${SHORT_SHA}-previous-k0s/g" e2e/kots-release-install/cluster-config.yaml - replicated release create --yaml-dir e2e/kots-release-install --promote CI-airgap --version "appver-${SHORT_SHA}-previous-k0s" + # return the cluster config to the current version + sed -i "s/${SHORT_SHA}-previous-k0s/${SHORT_SHA}/g" e2e/kots-release-install/cluster-config.yaml + + replicated release create --yaml-dir e2e/kots-release-install --promote CI-airgap --version "appver-${SHORT_SHA}" replicated release create --yaml-dir e2e/kots-release-upgrade --promote CI-airgap --version "appver-${SHORT_SHA}-upgrade" - name: Create download link message text @@ -316,6 +328,7 @@ jobs: - TestCommandsRequireSudo - TestInstallWithoutEmbed - TestInstallFromReplicatedApp + - TestUpgradeFromReplicatedApp - TestResetAndReinstall - TestResetAndReinstallAirgap - TestCollectSupportBundle @@ -367,6 +380,7 @@ jobs: license: ${{ secrets.STAGING_EMBEDDED_CLUSTER_LICENSE }} dr-aws-access-key-id: ${{ secrets.TESTIM_AWS_ACCESS_KEY_ID }} dr-aws-secret-access-key: ${{ secrets.TESTIM_AWS_SECRET_ACCESS_KEY }} + k0s-version: ${{ steps.build.outputs.k0s_version }} # this job will validate that all the tests passed # it is used for the github branch protection rule diff --git a/e2e/install_test.go b/e2e/install_test.go index 6cc48fcbb..b8a21774b 100644 --- a/e2e/install_test.go +++ b/e2e/install_test.go @@ -48,7 +48,7 @@ func TestSingleNodeInstallation(t *testing.T) { } t.Logf("%s: checking installation state after upgrade", time.Now().Format(time.RFC3339)) - line = []string{"check-postupgrade-state.sh", os.Getenv("SHORT_SHA")} + line = []string{"check-postupgrade-state.sh", k8sVersion()} if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil { t.Fatalf("fail to check postupgrade state: %v", err) } @@ -104,7 +104,7 @@ func TestSingleNodeInstallationAlmaLinux8(t *testing.T) { } t.Logf("%s: checking installation state after upgrade", time.Now().Format(time.RFC3339)) - line = []string{"check-postupgrade-state.sh", os.Getenv("SHORT_SHA")} + line = []string{"check-postupgrade-state.sh", k8sVersion()} if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil { t.Fatalf("fail to check postupgrade state: %v", err) } @@ -164,7 +164,7 @@ func TestSingleNodeInstallationDebian12(t *testing.T) { } t.Logf("%s: checking installation state after upgrade", time.Now().Format(time.RFC3339)) - line = []string{"check-postupgrade-state.sh", os.Getenv("SHORT_SHA")} + line = []string{"check-postupgrade-state.sh", k8sVersion()} if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil { t.Fatalf("fail to check postupgrade state: %v", err) } @@ -224,7 +224,7 @@ func TestSingleNodeInstallationDebian11(t *testing.T) { } t.Logf("%s: checking installation state after upgrade", time.Now().Format(time.RFC3339)) - line = []string{"check-postupgrade-state.sh", os.Getenv("SHORT_SHA")} + line = []string{"check-postupgrade-state.sh", k8sVersion()} if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil { t.Fatalf("fail to check postupgrade state: %v", err) } @@ -280,7 +280,7 @@ func TestSingleNodeInstallationCentos9Stream(t *testing.T) { } t.Logf("%s: checking installation state after upgrade", time.Now().Format(time.RFC3339)) - line = []string{"check-postupgrade-state.sh", os.Getenv("SHORT_SHA")} + line = []string{"check-postupgrade-state.sh", k8sVersion()} if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil { t.Fatalf("fail to check postupgrade state: %v", err) } @@ -499,7 +499,61 @@ func TestInstallFromReplicatedApp(t *testing.T) { } t.Logf("%s: checking installation state after upgrade", time.Now().Format(time.RFC3339)) - line = []string{"check-postupgrade-state.sh", os.Getenv("SHORT_SHA")} + line = []string{"check-postupgrade-state.sh", k8sVersion()} + if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil { + t.Fatalf("fail to check postupgrade state: %v", err) + } + + t.Logf("%s: test complete", time.Now().Format(time.RFC3339)) +} + +func TestUpgradeFromReplicatedApp(t *testing.T) { + t.Parallel() + + RequireEnvVars(t, []string{"SHORT_SHA"}) + + tc := cluster.NewTestCluster(&cluster.Input{ + T: t, + Nodes: 1, + Image: "debian/12", + }) + defer cleanupCluster(t, tc) + + t.Logf("%s: downloading embedded-cluster on node 0", time.Now().Format(time.RFC3339)) + line := []string{"vandoor-prepare.sh", fmt.Sprintf("%s-previous-k0s", os.Getenv("SHORT_SHA")), os.Getenv("LICENSE_ID"), "false"} + if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil { + t.Fatalf("fail to download embedded-cluster on node 0 %s: %v", tc.Nodes[0], err) + } + + t.Logf("%s: installing embedded-cluster on node 0", time.Now().Format(time.RFC3339)) + line = []string{"single-node-install.sh", "ui"} + if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil { + t.Fatalf("fail to install embedded-cluster on node %s: %v", tc.Nodes[0], err) + } + + if err := setupPlaywright(t, tc); err != nil { + t.Fatalf("fail to setup playwright: %v", err) + } + if _, _, err := runPlaywrightTest(t, tc, "deploy-app"); err != nil { + t.Fatalf("fail to run playwright test deploy-app: %v", err) + } + + t.Logf("%s: checking installation state", time.Now().Format(time.RFC3339)) + line = []string{"check-installation-state.sh", fmt.Sprintf("%s-previous-k0s", os.Getenv("SHORT_SHA"))} + if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil { + t.Fatalf("fail to check installation state: %v", err) + } + + appUpgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA")) + testArgs := []string{appUpgradeVersion} + + t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339)) + if _, _, err := runPlaywrightTest(t, tc, "deploy-upgrade", testArgs...); err != nil { + t.Fatalf("fail to run playwright test deploy-app: %v", err) + } + + t.Logf("%s: checking installation state after upgrade", time.Now().Format(time.RFC3339)) + line = []string{"check-postupgrade-state.sh", k8sVersion()} if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil { t.Fatalf("fail to check postupgrade state: %v", err) } @@ -653,7 +707,7 @@ func TestOldVersionUpgrade(t *testing.T) { } t.Logf("%s: checking installation state after upgrade", time.Now().Format(time.RFC3339)) - line = []string{"check-postupgrade-state.sh", os.Getenv("SHORT_SHA")} + line = []string{"check-postupgrade-state.sh", k8sVersion()} if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil { t.Fatalf("fail to check postupgrade state: %v", err) } @@ -763,7 +817,7 @@ func TestSingleNodeAirgapUpgrade(t *testing.T) { } t.Logf("%s: checking installation state after upgrade", time.Now().Format(time.RFC3339)) - line = []string{"check-postupgrade-state.sh", os.Getenv("SHORT_SHA")} + line = []string{"check-postupgrade-state.sh", k8sVersion()} if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil { t.Fatalf("fail to check postupgrade state: %v", err) } @@ -875,7 +929,7 @@ func TestSingleNodeAirgapUpgradeCustomCIDR(t *testing.T) { } t.Logf("%s: checking installation state after upgrade", time.Now().Format(time.RFC3339)) - line = []string{"check-postupgrade-state.sh", os.Getenv("SHORT_SHA")} + line = []string{"check-postupgrade-state.sh", k8sVersion()} if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil { t.Fatalf("fail to check postupgrade state: %v", err) } @@ -1055,7 +1109,7 @@ func TestMultiNodeAirgapUpgradeSameK0s(t *testing.T) { } t.Logf("%s: checking installation state after upgrade", time.Now().Format(time.RFC3339)) - line = []string{"check-postupgrade-state.sh", os.Getenv("SHORT_SHA")} + line = []string{"check-postupgrade-state.sh", k8sVersion()} if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil { t.Fatalf("fail to check postupgrade state: %v", err) } @@ -1219,7 +1273,7 @@ func TestMultiNodeAirgapUpgrade(t *testing.T) { } t.Logf("%s: checking installation state after upgrade", time.Now().Format(time.RFC3339)) - line = []string{"check-postupgrade-state.sh", os.Getenv("SHORT_SHA")} + line = []string{"check-postupgrade-state.sh", k8sVersion()} if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil { t.Fatalf("fail to check postupgrade state: %v", err) } @@ -1334,7 +1388,7 @@ func TestMultiNodeHAInstallation(t *testing.T) { } t.Logf("%s: checking installation state after upgrade", time.Now().Format(time.RFC3339)) - line = []string{"check-postupgrade-state.sh", os.Getenv("SHORT_SHA")} + line = []string{"check-postupgrade-state.sh", k8sVersion()} if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil { t.Fatalf("fail to check postupgrade state: %v", err) } @@ -1546,7 +1600,7 @@ func TestMultiNodeAirgapHAInstallation(t *testing.T) { } t.Logf("%s: checking installation state after upgrade", time.Now().Format(time.RFC3339)) - line = []string{"check-postupgrade-state.sh", os.Getenv("SHORT_SHA")} + line = []string{"check-postupgrade-state.sh", k8sVersion()} if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil { t.Fatalf("fail to check postupgrade state: %v", err) } @@ -1606,7 +1660,7 @@ func TestInstallSnapshotFromReplicatedApp(t *testing.T) { } t.Logf("%s: checking installation state after upgrade", time.Now().Format(time.RFC3339)) - line = []string{"check-postupgrade-state.sh", os.Getenv("SHORT_SHA")} + line = []string{"check-postupgrade-state.sh", k8sVersion()} if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil { t.Fatalf("fail to check postupgrade state: %v", err) } diff --git a/e2e/restore_test.go b/e2e/restore_test.go index 7a3b053a2..2357530f7 100644 --- a/e2e/restore_test.go +++ b/e2e/restore_test.go @@ -390,7 +390,7 @@ func TestSingleNodeAirgapDisasterRecovery(t *testing.T) { } t.Logf("%s: checking installation state after upgrade", time.Now().Format(time.RFC3339)) - line = []string{"check-postupgrade-state.sh"} + line = []string{"check-postupgrade-state.sh", k8sVersion()} if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil { t.Fatalf("fail to check postupgrade state: %v", err) } @@ -591,7 +591,7 @@ func TestMultiNodeHADisasterRecovery(t *testing.T) { } t.Logf("%s: checking installation state after upgrade", time.Now().Format(time.RFC3339)) - line = []string{"check-postupgrade-state.sh", os.Getenv("SHORT_SHA")} + line = []string{"check-postupgrade-state.sh", k8sVersion()} if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil { t.Fatalf("fail to check postupgrade state: %v", err) } diff --git a/e2e/scripts/check-postupgrade-state.sh b/e2e/scripts/check-postupgrade-state.sh index f75b69ba6..67bd2d14b 100755 --- a/e2e/scripts/check-postupgrade-state.sh +++ b/e2e/scripts/check-postupgrade-state.sh @@ -12,6 +12,8 @@ function check_nginx_version { } main() { + local k8s_version="$1" + echo "ensure that installation is installed" wait_for_installation @@ -102,6 +104,12 @@ main() { echo "no charts had an order of '110'" exit 1 fi + + echo "ensure that all nodes are running k8s $k8s_version" + if ! ensure_nodes_match_kube_version "$k8s_version"; then + echo "not all nodes are running k8s $k8s_version" + exit 1 + fi } export EMBEDDED_CLUSTER_METRICS_BASEURL="https://staging.replicated.app" diff --git a/e2e/scripts/common.sh b/e2e/scripts/common.sh index 88ae95dbb..9659d370f 100644 --- a/e2e/scripts/common.sh +++ b/e2e/scripts/common.sh @@ -203,6 +203,16 @@ ensure_node_config() { fi } +ensure_nodes_match_kube_version() { + local version="$1" + if kubectl get nodes -o jsonpath='{.items[*].status.nodeInfo.kubeletVersion}' | grep -v "$version"; then + echo "Node kubelet version does not match expected version $version" + kubectl get nodes -o jsonpath='{.items[*].status.nodeInfo.kubeletVersion}' + kubectl get nodes + return 1 + fi +} + check_pod_install_order() { local ingress_install_time= ingress_install_time=$(kubectl get pods --no-headers=true -n ingress-nginx -o jsonpath='{.items[*].metadata.creationTimestamp}' | sort | head -n 1) diff --git a/e2e/utils.go b/e2e/utils.go index 57fc9de72..0e17b9f82 100644 --- a/e2e/utils.go +++ b/e2e/utils.go @@ -159,3 +159,12 @@ func injectString(original, injection, after string) string { // Construct the new string by adding the injection between the parts return parts[0] + after + " " + injection + parts[1] } + +func k8sVersion() string { + // split the version string (like 'v1.29.6+k0s.0') into the k8s version and the k0s revision + verParts := strings.Split(os.Getenv("K0S_VERSION"), "+") + if len(verParts) < 2 { + panic(fmt.Sprintf("failed to parse k8s version %q", os.Getenv("K0S_VERSION"))) + } + return verParts[0] +} diff --git a/pkg/addons/embeddedclusteroperator/static/metadata.yaml b/pkg/addons/embeddedclusteroperator/static/metadata.yaml index c851ddfdc..472ab8084 100644 --- a/pkg/addons/embeddedclusteroperator/static/metadata.yaml +++ b/pkg/addons/embeddedclusteroperator/static/metadata.yaml @@ -5,8 +5,8 @@ # $ make buildtools # $ output/bin/buildtools update addon # -version: 0.42.0 +version: 0.42.1 location: oci://proxy.replicated.com/anonymous/registry.replicated.com/library/embedded-cluster-operator images: - replicated/embedded-cluster-operator-image: 0.42.0@sha256:a0eab9ea8bb058f392c75d34b6fa88ae66b3f5fc6ab8953686379cacb13bb11f + replicated/embedded-cluster-operator-image: 0.42.1@sha256:bc293473dd59a03f82e1328bed58c6ba24b8fbdaacc4559b8c338c933239a042 utils: latest@sha256:88c0e2ba4daa9682ca85de32e11aad545d338cdaf91edf6ab52eb0c2d6b825a3 diff --git a/pkg/addons/embeddedclusteroperator/static/values.yaml b/pkg/addons/embeddedclusteroperator/static/values.yaml index 4a7aa4d8a..3a673dd19 100644 --- a/pkg/addons/embeddedclusteroperator/static/values.yaml +++ b/pkg/addons/embeddedclusteroperator/static/values.yaml @@ -17,5 +17,5 @@ global: replicated.com/disaster-recovery-chart: embedded-cluster-operator image: repository: proxy.replicated.com/anonymous/replicated/embedded-cluster-operator-image - tag: 0.42.0@sha256:a0eab9ea8bb058f392c75d34b6fa88ae66b3f5fc6ab8953686379cacb13bb11f + tag: 0.42.1@sha256:bc293473dd59a03f82e1328bed58c6ba24b8fbdaacc4559b8c338c933239a042 utilsImage: 'proxy.replicated.com/anonymous/replicated/ec-utils@sha256:88c0e2ba4daa9682ca85de32e11aad545d338cdaf91edf6ab52eb0c2d6b825a3'