feat: v2 migration e2e tests

.github/workflows/ci.yaml

@@ -473,6 +473,77 @@ jobs:
           echo "K0S_VERSION=\"$K0S_VERSION\""
           echo "k0s_version=$K0S_VERSION" >> "$GITHUB_OUTPUT"
 
+  build-upgrade-v2:
+    name: Build upgrade v2
+    runs-on: embedded-cluster-2
+    needs:
+      - git-sha
+    outputs:
+      k0s_version: ${{ steps.export.outputs.k0s_version }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Cache embedded bins
+        uses: actions/cache@v4
+        with:
+          path: |
+            output/bins
+          key: bins-cache
+
+      - name: Setup go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+          cache-dependency-path: "**/*.sum"
+
+      - name: Install dagger
+        run: |
+          curl -fsSL https://dl.dagger.io/dagger/install.sh | DAGGER_VERSION=v0.14.0 sh
+          sudo mv ./bin/dagger /usr/local/bin/dagger
+
+      - name: Build
+        env:
+          APP_CHANNEL_ID: 2cHXb1RCttzpR0xvnNWyaZCgDBP
+          APP_CHANNEL_SLUG: ci
+          RELEASE_YAML_DIR: e2e/kots-release-upgrade
+          S3_BUCKET: "tf-staging-embedded-cluster-bin"
+          USES_DEV_BUCKET: "0"
+          AWS_ACCESS_KEY_ID: ${{ secrets.STAGING_EMBEDDED_CLUSTER_UPLOAD_IAM_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.STAGING_EMBEDDED_CLUSTER_UPLOAD_IAM_SECRET }}
+          AWS_REGION: "us-east-1"
+          USE_CHAINGUARD: "1"
+          UPLOAD_BINARIES: "1"
+          SKIP_RELEASE: "1"
+          MANGLE_METADATA: "1"
+          V2_ENABLED: "1"
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          export K0S_VERSION=$(make print-K0S_VERSION)
+          export EC_VERSION=$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-v2upgrade
+          export APP_VERSION=appver-dev-${{ needs.git-sha.outputs.git_sha }}-v2upgrade
+          # avoid rate limiting
+          export FIO_VERSION=$(gh release list --repo axboe/fio --json tagName,isLatest | jq -r '.[] | select(.isLatest==true)|.tagName' | cut -d- -f2)
+
+          ./scripts/build-and-release.sh
+          cp output/bin/embedded-cluster output/bin/embedded-cluster-v2upgrade
+
+      - name: Upload release
+        uses: actions/upload-artifact@v4
+        with:
+          name: upgrade-v2-release
+          path: |
+            output/bin/embedded-cluster-v2upgrade
+
+      - name: Export k0s version
+        id: export
+        run: |
+          K0S_VERSION="$(make print-K0S_VERSION)"
+          echo "K0S_VERSION=\"$K0S_VERSION\""
+          echo "k0s_version=$K0S_VERSION" >> "$GITHUB_OUTPUT"
+
   check-images:
     name: Check images
     runs-on: ubuntu-latest
@@ -537,6 +608,7 @@ jobs:
       - build-legacydr
       - build-previous-k0s
       - build-upgrade
+      - build-upgrade-v2
       - find-previous-stable
     steps:
       - name: Checkout
@@ -600,6 +672,14 @@ jobs:
           export RELEASE_YAML_DIR=e2e/kots-release-install
           ./scripts/ci-release-app.sh
 
+          # an app upgrade to v2
+          export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-v2upgrade"
+          export APP_VERSION="appver-${SHORT_SHA}-v2upgrade"
+          export V2_ENABLED="1"
+          export RELEASE_YAML_DIR=e2e/kots-release-upgrade
+          ./scripts/ci-release-app.sh
+          unset V2_ENABLED
+
           # and finally an app upgrade
           export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-upgrade"
           export APP_VERSION="appver-${SHORT_SHA}-upgrade"
@@ -640,6 +720,14 @@ jobs:
           export RELEASE_YAML_DIR=e2e/kots-release-install-legacydr
           ./scripts/ci-release-app.sh
 
+          # an app upgrade to v2
+          export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-v2upgrade"
+          export APP_VERSION="appver-${SHORT_SHA}-v2upgrade"
+          export V2_ENABLED="1"
+          export RELEASE_YAML_DIR=e2e/kots-release-upgrade
+          ./scripts/ci-release-app.sh
+          unset V2_ENABLED
+
           # and finally an app upgrade
           export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-upgrade"
           export APP_VERSION="appver-${SHORT_SHA}-upgrade"
@@ -682,6 +770,7 @@ jobs:
       - build-legacydr
       - build-previous-k0s
       - build-upgrade
+      - build-upgrade-v2
       - find-previous-stable
       - release-app
       - export-version-specifier
@@ -701,6 +790,7 @@ jobs:
           - TestSingleNodeUpgradePreviousStable
           - TestInstallFromReplicatedApp
           - TestUpgradeFromReplicatedApp
+          - TestMigrateV2FromReplicatedApp
           - TestInstallWithoutEmbed
           - TestUpgradeEC18FromReplicatedApp
           - TestResetAndReinstall
@@ -782,6 +872,7 @@ jobs:
       - build-legacydr
       - build-previous-k0s
       - build-upgrade
+      - build-upgrade-v2
       - find-previous-stable
       - release-app
       - export-version-specifier
@@ -850,6 +941,7 @@ jobs:
       - build-legacydr
       - build-previous-k0s
       - build-upgrade
+      - build-upgrade-v2
       - find-previous-stable
       - release-app
       - export-version-specifier

cmd/local-artifact-mirror/artifact.go

@@ -2,129 +2,35 @@ package main
 
 import (
 	"context"
-	"crypto/tls"
-	"encoding/json"
 	"fmt"
-	"net/http"
 	"os"
 
+	"github.com/replicatedhq/embedded-cluster/pkg/registry"
 	"github.com/sirupsen/logrus"
 	"go.uber.org/multierr"
-	corev1 "k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/api/errors"
-	"k8s.io/apimachinery/pkg/types"
-	"oras.land/oras-go/v2"
-	"oras.land/oras-go/v2/content/file"
-	"oras.land/oras-go/v2/registry"
-	"oras.land/oras-go/v2/registry/remote"
-	"oras.land/oras-go/v2/registry/remote/auth"
-	"oras.land/oras-go/v2/registry/remote/credentials"
 )
 
-var (
-	insecureTransport *http.Transport
-)
-
-func init() {
-	insecureTransport = http.DefaultTransport.(*http.Transport).Clone()
-	insecureTransport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
-}
-
-// DockerConfig represents the content of the '.dockerconfigjson' secret.
-type DockerConfig struct {
-	Auths map[string]DockerConfigEntry `json:"auths"`
-}
-
-// DockerConfigEntry represents the content of the '.dockerconfigjson' secret.
-type DockerConfigEntry struct {
-	Username string `json:"username"`
-	Password string `json:"password"`
-}
-
-// registryAuth returns the authentication store to be used when reaching the
-// registry. The authentication store is read from the cluster secret named
-// 'registry-creds' in the 'kotsadm' namespace.
-func registryAuth(ctx context.Context) (credentials.Store, error) {
-	nsn := types.NamespacedName{Name: "registry-creds", Namespace: "kotsadm"}
-	var sct corev1.Secret
-	if err := kubecli.Get(ctx, nsn, &sct); err != nil {
-		if !errors.IsNotFound(err) {
-			return nil, fmt.Errorf("unable to get secret: %w", err)
-		}
-
-		// if we can't locate a secret then returns an empty credentials
-		// store so we attempt to fetch the assets without auth.
-		logrus.Infof("no registry auth found, trying anonymous access")
-		return credentials.NewMemoryStore(), nil
-	}
-
-	data, ok := sct.Data[".dockerconfigjson"]
-	if !ok {
-		return nil, fmt.Errorf("unable to find secret .dockerconfigjson")
-	}
-
-	var cfg DockerConfig
-	if err := json.Unmarshal(data, &cfg); err != nil {
-		return nil, fmt.Errorf("unable to unmarshal secret: %w", err)
-	}
-
-	creds := credentials.NewMemoryStore()
-	for addr, entry := range cfg.Auths {
-		creds.Put(ctx, addr, auth.Credential{
-			Username: entry.Username,
-			Password: entry.Password,
-		})
-	}
-	return creds, nil
-}
-
 // pullArtifact fetches an artifact from the registry pointed by 'from'. The artifact
 // is stored in a temporary directory and the path to this directory is returned.
 // Callers are responsible for removing the temporary directory when it is no longer
 // needed. In case of error, the temporary directory is removed here.
 func pullArtifact(ctx context.Context, from string) (string, error) {
-	store, err := registryAuth(ctx)
-	if err != nil {
-		return "", fmt.Errorf("unable to get registry auth: %w", err)
-	}
-
-	imgref, err := registry.ParseReference(from)
-	if err != nil {
-		return "", fmt.Errorf("unable to parse image reference: %w", err)
-	}
-
 	tmpdir, err := os.MkdirTemp("", "embedded-cluster-artifact-*")
 	if err != nil {
 		return "", fmt.Errorf("unable to create temp dir: %w", err)
 	}
 
-	repo, err := remote.NewRepository(from)
-	if err != nil {
-		return "", fmt.Errorf("unable to create repository: %w", err)
-	}
-
-	fs, err := file.New(tmpdir)
-	if err != nil {
-		return "", fmt.Errorf("unable to create file store: %w", err)
-	}
-	defer fs.Close()
-
-	repo.Client = &auth.Client{
-		Client:     &http.Client{Transport: insecureTransport},
-		Credential: store.Get,
-	}
-
-	tag := imgref.Reference
-	_, tlserr := oras.Copy(ctx, repo, tag, fs, tag, oras.DefaultCopyOptions)
+	opts := registry.PullArtifactOptions{}
+	tlserr := registry.PullArtifact(ctx, kubecli, from, tmpdir, opts)
 	if tlserr == nil {
 		return tmpdir, nil
 	}
 
 	// if we fail to fetch the artifact using https we gonna try once more using plain
 	// http as some versions of the registry were deployed without tls.
-	repo.PlainHTTP = true
+	opts.PlainHTTP = true
 	logrus.Infof("unable to fetch artifact using tls, retrying with http")
-	if _, err := oras.Copy(ctx, repo, tag, fs, tag, oras.DefaultCopyOptions); err != nil {
+	if err := registry.PullArtifact(ctx, kubecli, from, tmpdir, opts); err != nil {
 		os.RemoveAll(tmpdir)
 		err = multierr.Combine(tlserr, err)
 		return "", fmt.Errorf("unable to fetch artifacts with or without tls: %w", err)

dev/dockerfiles/operator/Dockerfile.ttlsh

@@ -18,6 +18,7 @@ ENV VERSION=${VERSION}
 ARG K0S_VERSION
 ENV K0S_VERSION=${K0S_VERSION}
 
+ENV GOCACHE=/root/.cache/go-build
 RUN --mount=type=cache,target="/root/.cache/go-build" make -C operator build
 
 FROM debian:bookworm-slim

e2e/install_test.go

@@ -487,6 +487,91 @@ func TestInstallFromReplicatedApp(t *testing.T) {
 	t.Logf("%s: test complete", time.Now().Format(time.RFC3339))
 }
 
+func TestMigrateV2FromReplicatedApp(t *testing.T) {
+	t.Parallel()
+
+	RequireEnvVars(t, []string{"SHORT_SHA"})
+
+	tc := docker.NewCluster(&docker.ClusterInput{
+		T:      t,
+		Nodes:  1,
+		Distro: "debian-bookworm",
+	})
+	defer tc.Cleanup()
+
+	t.Logf("%s: downloading embedded-cluster on node 0", time.Now().Format(time.RFC3339))
+	line := []string{"vandoor-prepare.sh", fmt.Sprintf("appver-%s", os.Getenv("SHORT_SHA")), os.Getenv("LICENSE_ID"), "false"}
+	if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil {
+		t.Fatalf("fail to download embedded-cluster on node 0: %v: %s: %s", err, stdout, stderr)
+	}
+
+	t.Logf("%s: installing embedded-cluster on node 0", time.Now().Format(time.RFC3339))
+	line = []string{"single-node-install.sh", "ui", os.Getenv("SHORT_SHA")}
+	if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil {
+		t.Fatalf("fail to install embedded-cluster on node 0: %v: %s: %s", err, stdout, stderr)
+	}
+
+	if stdout, stderr, err := tc.SetupPlaywrightAndRunTest("deploy-app"); err != nil {
+		t.Fatalf("fail to run playwright test deploy-app: %v: %s: %s", err, stdout, stderr)
+	}
+
+	t.Logf("%s: checking installation state", time.Now().Format(time.RFC3339))
+	line = []string{"check-installation-state.sh", os.Getenv("SHORT_SHA"), k8sVersion()}
+	if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil {
+		t.Fatalf("fail to check installation state: %v: %s: %s", err, stdout, stderr)
+	}
+
+	// upgrade the cluster and migrate to v2
+
+	appUpgradeVersion := fmt.Sprintf("appver-%s-v2upgrade", os.Getenv("SHORT_SHA"))
+	testArgs := []string{appUpgradeVersion}
+
+	t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339))
+	if stdout, stderr, err := tc.RunPlaywrightTest("deploy-upgrade", testArgs...); err != nil {
+		t.Fatalf("fail to run playwright test deploy-app: %v: %s: %s", err, stdout, stderr)
+	}
+
+	t.Logf("%s: checking installation state after upgrade and v2 migration", time.Now().Format(time.RFC3339))
+	line = []string{"check-postupgrade-state.sh", k8sVersion(), ecUpgradeTargetVersion(), "true"}
+	if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil {
+		t.Fatalf("fail to check postupgrade state: %v: %s: %s", err, stdout, stderr)
+	}
+
+	t.Logf("%s: checking installation state after migration to v2", time.Now().Format(time.RFC3339))
+	line = []string{"check-postv2migration-state.sh"}
+	if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil {
+		t.Fatalf("fail to check post v2 migration state: %v: %s: %s", err, stdout, stderr)
+	}
+
+	// upgrade the cluster one more time post v2 migration
+
+	appUpgradeVersion = fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA"))
+	testArgs = []string{appUpgradeVersion}
+
+	t.Logf("%s: upgrading cluster again", time.Now().Format(time.RFC3339))
+	if stdout, stderr, err := tc.RunPlaywrightTest("deploy-upgrade", testArgs...); err != nil {
+		t.Fatalf("fail to run playwright test deploy-app: %v: %s: %s", err, stdout, stderr)
+	}
+
+	t.Logf("%s: checking postuprgrade state for an install2 cluster", time.Now().Format(time.RFC3339))
+	line = []string{"check-postupgrade-state2.sh", appUpgradeVersion, k8sVersion()}
+	if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil {
+		t.Fatalf("fail to check installation state: %v: %s: %s", err, stdout, stderr)
+	}
+
+	t.Logf("%s: resetting admin console password", time.Now().Format(time.RFC3339))
+	newPassword := "newpass"
+	line = []string{"embedded-cluster", "admin-console", "reset-password", newPassword}
+	_, _, err := tc.RunCommandOnNode(0, line)
+	require.NoError(t, err, "unable to reset admin console password")
+
+	t.Logf("%s: logging in with the new password", time.Now().Format(time.RFC3339))
+	_, _, err = tc.RunPlaywrightTest("login-with-custom-password", newPassword)
+	require.NoError(t, err, "unable to login with the new password")
+
+	t.Logf("%s: test complete", time.Now().Format(time.RFC3339))
+}
+
 func TestSingleNodeUpgradePreviousStable(t *testing.T) {
 	t.Parallel()
 

e2e/playwright/tests/deploy-upgrade/test.spec.ts

@@ -22,7 +22,8 @@ async function fillConfigForm(iframe: FrameLocator) {
   await expect(iframe.locator('h3')).toContainText('The First Config Group', { timeout: 60 * 1000 }); // can take time to download the kots binary
 
   const hostnameInput = iframe.locator('#hostname-group').locator('input[type="text"]');
-  await expect(hostnameInput).toHaveValue(process.env.APP_INITIAL_HOSTNAME ? process.env.APP_INITIAL_HOSTNAME : 'initial-hostname.com');
+  // the hostname can be either 'initial-hostname.com' or 'updated-hostname.com' if we have run the upgrade multiple times
+  await expect(hostnameInput).toHaveValue(process.env.APP_INITIAL_HOSTNAME ? process.env.APP_INITIAL_HOSTNAME : /(initial|updated)-hostname\.com/);
   await hostnameInput.click();
   await hostnameInput.fill('updated-hostname.com');