From 0ba8dac2a3365192935cb2e34d77817bdf165ca2 Mon Sep 17 00:00:00 2001
From: Salah Al Saleh <sg.alsaleh@gmail.com>
Date: Mon, 24 Jun 2024 20:58:06 +0000
Subject: [PATCH 01/96] Airgap updates via CLI

---
 cmd/embedded-cluster/update.go |  2 +-
 pkg/kotscli/kotscli.go         | 23 ++++++++++-------------
 2 files changed, 11 insertions(+), 14 deletions(-)

diff --git a/cmd/embedded-cluster/update.go b/cmd/embedded-cluster/update.go
index 9586b2858..cfccce2a0 100644
--- a/cmd/embedded-cluster/update.go
+++ b/cmd/embedded-cluster/update.go
@@ -46,7 +46,7 @@ var updateCommand = &cli.Command{
 			return fmt.Errorf("no channel release found")
 		}
 
-		if err := kotscli.UpstreamUpgrade(kotscli.UpstreamUpgradeOptions{
+		if err := kotscli.AirgapUpdate(kotscli.AirgapUpdateOptions{
 			AppSlug:      rel.AppSlug,
 			Namespace:    defaults.KotsadmNamespace,
 			AirgapBundle: c.String("airgap-bundle"),
diff --git a/pkg/kotscli/kotscli.go b/pkg/kotscli/kotscli.go
index 14abe7e22..3ca072836 100644
--- a/pkg/kotscli/kotscli.go
+++ b/pkg/kotscli/kotscli.go
@@ -82,32 +82,29 @@ func Install(opts InstallOptions, msg *spinner.MessageWriter) error {
 	return nil
 }
 
-type UpstreamUpgradeOptions struct {
+type AirgapUpdateOptions struct {
 	AppSlug      string
 	Namespace    string
 	AirgapBundle string
 }
 
-func UpstreamUpgrade(opts UpstreamUpgradeOptions) error {
+func AirgapUpdate(opts AirgapUpdateOptions) error {
 	kotsBinPath, err := goods.MaterializeInternalBinary("kubectl-kots")
 	if err != nil {
 		return fmt.Errorf("unable to materialize kubectl-kots binary: %w", err)
 	}
 	defer os.Remove(kotsBinPath)
 
-	var lbreakfn spinner.LineBreakerFn
-	maskfn := MaskKotsOutputForOnline()
-	upstreamUpgradeArgs := []string{
-		"upstream",
-		"upgrade",
+	maskfn := MaskKotsOutputForAirgap()
+	lbreakfn := KotsOutputLineBreaker()
+
+	airgapUpdateArgs := []string{
+		"airgap-update",
 		opts.AppSlug,
 		"--namespace",
 		opts.Namespace,
-	}
-	if opts.AirgapBundle != "" {
-		upstreamUpgradeArgs = append(upstreamUpgradeArgs, "--airgap-bundle", opts.AirgapBundle)
-		maskfn = MaskKotsOutputForAirgap()
-		lbreakfn = KotsOutputLineBreaker()
+		"--airgap-bundle",
+		opts.AirgapBundle,
 	}
 
 	loading := spinner.Start(spinner.WithMask(maskfn), spinner.WithLineBreaker(lbreakfn))
@@ -117,7 +114,7 @@ func UpstreamUpgrade(opts UpstreamUpgradeOptions) error {
 			"EMBEDDED_CLUSTER_ID": metrics.ClusterID().String(),
 		},
 	}
-	if err := helpers.RunCommandWithOptions(runCommandOptions, kotsBinPath, upstreamUpgradeArgs...); err != nil {
+	if err := helpers.RunCommandWithOptions(runCommandOptions, kotsBinPath, airgapUpdateArgs...); err != nil {
 		loading.CloseWithError()
 		return fmt.Errorf("unable to update the application: %w", err)
 	}

From 25038ffeed271b8656f4673ad8aed95bdc0733d3 Mon Sep 17 00:00:00 2001
From: Salah Al Saleh <sg.alsaleh@gmail.com>
Date: Mon, 24 Jun 2024 21:44:45 +0000
Subject: [PATCH 02/96] update messages

---
 pkg/kotscli/kotscli.go | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/pkg/kotscli/kotscli.go b/pkg/kotscli/kotscli.go
index 3ca072836..96962249d 100644
--- a/pkg/kotscli/kotscli.go
+++ b/pkg/kotscli/kotscli.go
@@ -174,8 +174,8 @@ func VeleroConfigureOtherS3(opts VeleroConfigureOtherS3Options) error {
 }
 
 // MaskKotsOutputForOnline masks the kots cli output during online installations. For
-// online installations we only want to print "Finalizing" until it is done and then
-// print "Finished!".
+// online installations we only want to print "Finalizing Admin Console" until it is done
+// and then print "Finished!".
 func MaskKotsOutputForOnline() spinner.MaskFn {
 	return func(message string) string {
 		if strings.Contains(message, "Finished") {
@@ -189,13 +189,15 @@ func MaskKotsOutputForOnline() spinner.MaskFn {
 // function replaces some of the messages being printed to the user so the output looks
 // nicer.
 func MaskKotsOutputForAirgap() spinner.MaskFn {
-	current := "Uploading air gap bundle"
+	current := "Extracting air gap bundle"
 	return func(message string) string {
 		switch {
 		case strings.Contains(message, "Pushing application images"):
 			current = message
 		case strings.Contains(message, "Pushing embedded cluster artifacts"):
 			current = message
+		case strings.Contains(message, "Uploading airgap update"):
+			current = message
 		case strings.Contains(message, "Waiting for the Admin Console"):
 			current = "Finalizing Admin Console"
 		case strings.Contains(message, "Finished!"):
@@ -259,7 +261,7 @@ func KotsOutputLineBreaker() spinner.LineBreakerFn {
 		// if we are printing a message about the finalization of the installation it
 		// means that the embedded cluster artifacts are ready and we want to break the
 		// line.
-		if current == "Finalizing" {
+		if strings.Contains(current, "Finalizing") {
 			return true, "Embedded cluster artifacts are ready!"
 		}
 		return false, ""

From 972b6ceb91a3873781037621c5dfd3b295b55125 Mon Sep 17 00:00:00 2001
From: Salah Al Saleh <sg.alsaleh@gmail.com>
Date: Wed, 26 Jun 2024 13:54:39 +0000
Subject: [PATCH 03/96] alpha build

---
 Makefile                                |  4 ++-
 e2e/kots-release-install/preflight.yaml | 34 +++++++++++++++++++++++++
 e2e/kots-release-upgrade/preflight.yaml | 34 +++++++++++++++++++++++++
 pkg/addons/adminconsole/adminconsole.go |  7 +++++
 scripts/cache-files.sh                  |  7 ++---
 5 files changed, 82 insertions(+), 4 deletions(-)
 create mode 100644 e2e/kots-release-install/preflight.yaml
 create mode 100644 e2e/kots-release-upgrade/preflight.yaml

diff --git a/Makefile b/Makefile
index 4f6e6e966..176db56f3 100644
--- a/Makefile
+++ b/Makefile
@@ -4,8 +4,9 @@ ARCH := $(shell uname -m)
 APP_NAME = embedded-cluster
 ADMIN_CONSOLE_CHART_REPO_OVERRIDE =
 ADMIN_CONSOLE_CHART_VERSION = 1.111.0
-ADMIN_CONSOLE_IMAGE_OVERRIDE =
+ADMIN_CONSOLE_IMAGE_OVERRIDE = ttl.sh/sgalsaleh/kotsadm:24h
 ADMIN_CONSOLE_MIGRATIONS_IMAGE_OVERRIDE =
+ADMIN_CONSOLE_KURL_PROXY_IMAGE_OVERRIDE = ttl.sh/sgalsaleh/kurl-proxy:24h
 EMBEDDED_OPERATOR_CHART_URL = oci://registry.replicated.com/library
 EMBEDDED_OPERATOR_CHART_NAME = embedded-cluster-operator
 EMBEDDED_OPERATOR_CHART_VERSION = 0.36.5
@@ -39,6 +40,7 @@ LD_FLAGS = -X github.com/replicatedhq/embedded-cluster/pkg/defaults.K0sVersion=$
 	-X github.com/replicatedhq/embedded-cluster/pkg/addons/adminconsole.Version=$(ADMIN_CONSOLE_CHART_VERSION) \
 	-X github.com/replicatedhq/embedded-cluster/pkg/addons/adminconsole.ImageOverride=$(ADMIN_CONSOLE_IMAGE_OVERRIDE) \
 	-X github.com/replicatedhq/embedded-cluster/pkg/addons/adminconsole.MigrationsImageOverride=$(ADMIN_CONSOLE_MIGRATIONS_IMAGE_OVERRIDE) \
+	-X github.com/replicatedhq/embedded-cluster/pkg/addons/adminconsole.KurlProxyImageOverride=$(ADMIN_CONSOLE_KURL_PROXY_IMAGE_OVERRIDE) \
 	-X github.com/replicatedhq/embedded-cluster/pkg/addons/adminconsole.KotsVersion=$(KOTS_VERSION) \
 	-X github.com/replicatedhq/embedded-cluster/pkg/addons/embeddedclusteroperator.Version=$(EMBEDDED_OPERATOR_CHART_VERSION) \
 	-X github.com/replicatedhq/embedded-cluster/pkg/addons/embeddedclusteroperator.UtilsImage=$(EMBEDDED_OPERATOR_UTILS_IMAGE) \
diff --git a/e2e/kots-release-install/preflight.yaml b/e2e/kots-release-install/preflight.yaml
new file mode 100644
index 000000000..1c1fa9376
--- /dev/null
+++ b/e2e/kots-release-install/preflight.yaml
@@ -0,0 +1,34 @@
+apiVersion: troubleshoot.replicated.com/v1beta1
+kind: Preflight
+metadata:
+  name: preflight-checks
+spec:
+  analyzers:
+    - clusterVersion:
+        outcomes:
+          - fail:
+              when: "< 1.28.0"
+              message: The application requires at Kubernetes 1.28.0 or later, and recommends 1.29.0.
+              uri: https://www.kubernetes.io
+          - warn:
+              when: "< 1.29.0"
+              message: Your cluster meets the minimum version of Kubernetes, but we recommend you update to 1.29.0 or later.
+              uri: https://kubernetes.io
+          - pass:
+              message: Your cluster meets the recommended and required versions of Kubernetes.
+    - customResourceDefinition:
+        customResourceDefinitionName: installations.embeddedcluster.replicated.com
+        checkName: Embedded Cluster Installation CRD exists
+        outcomes:
+          - fail:
+              message: The Embedded Cluster Installation CRD is not installed in the cluster. Please install the Embedded Cluster Operator.
+          - pass:
+              message: The Embedded Cluster Installation CRD is installed in the cluster.
+    - customResourceDefinition:
+        customResourceDefinitionName: configs.embeddedcluster.replicated.com
+        checkName: Embedded Cluster Config CRD exists
+        outcomes:
+          - fail:
+              message: The Embedded Cluster Config CRD is not installed in the cluster. Please install the Embedded Cluster Operator.
+          - pass:
+              message: The Embedded Cluster Config CRD is installed in the cluster.
diff --git a/e2e/kots-release-upgrade/preflight.yaml b/e2e/kots-release-upgrade/preflight.yaml
new file mode 100644
index 000000000..1c1fa9376
--- /dev/null
+++ b/e2e/kots-release-upgrade/preflight.yaml
@@ -0,0 +1,34 @@
+apiVersion: troubleshoot.replicated.com/v1beta1
+kind: Preflight
+metadata:
+  name: preflight-checks
+spec:
+  analyzers:
+    - clusterVersion:
+        outcomes:
+          - fail:
+              when: "< 1.28.0"
+              message: The application requires at Kubernetes 1.28.0 or later, and recommends 1.29.0.
+              uri: https://www.kubernetes.io
+          - warn:
+              when: "< 1.29.0"
+              message: Your cluster meets the minimum version of Kubernetes, but we recommend you update to 1.29.0 or later.
+              uri: https://kubernetes.io
+          - pass:
+              message: Your cluster meets the recommended and required versions of Kubernetes.
+    - customResourceDefinition:
+        customResourceDefinitionName: installations.embeddedcluster.replicated.com
+        checkName: Embedded Cluster Installation CRD exists
+        outcomes:
+          - fail:
+              message: The Embedded Cluster Installation CRD is not installed in the cluster. Please install the Embedded Cluster Operator.
+          - pass:
+              message: The Embedded Cluster Installation CRD is installed in the cluster.
+    - customResourceDefinition:
+        customResourceDefinitionName: configs.embeddedcluster.replicated.com
+        checkName: Embedded Cluster Config CRD exists
+        outcomes:
+          - fail:
+              message: The Embedded Cluster Config CRD is not installed in the cluster. Please install the Embedded Cluster Operator.
+          - pass:
+              message: The Embedded Cluster Config CRD is installed in the cluster.
diff --git a/pkg/addons/adminconsole/adminconsole.go b/pkg/addons/adminconsole/adminconsole.go
index 82060f80d..d4de41f12 100644
--- a/pkg/addons/adminconsole/adminconsole.go
+++ b/pkg/addons/adminconsole/adminconsole.go
@@ -41,6 +41,7 @@ var (
 	Version                 = "v0.0.0"
 	ImageOverride           = ""
 	MigrationsImageOverride = ""
+	KurlProxyImageOverride  = ""
 	KotsVersion             = ""
 	CounterRegex            = regexp.MustCompile(`(\d+)/(\d+)`)
 )
@@ -84,6 +85,12 @@ func init() {
 		}
 		helmValues["images"].(map[string]interface{})["migrations"] = MigrationsImageOverride
 	}
+	if KurlProxyImageOverride != "" {
+		if helmValues["images"] == nil {
+			helmValues["images"] = map[string]interface{}{}
+		}
+		helmValues["images"].(map[string]interface{})["kurlProxy"] = KurlProxyImageOverride
+	}
 }
 
 // AdminConsole manages the admin console helm chart installation.
diff --git a/scripts/cache-files.sh b/scripts/cache-files.sh
index 60cc56f35..fdb4dc303 100755
--- a/scripts/cache-files.sh
+++ b/scripts/cache-files.sh
@@ -96,9 +96,10 @@ function operatorbin() {
 function kotsbin() {
     # first, figure out what version of kots is in the current build
     local kots_version=
-    kots_version=$(awk '/^ADMIN_CONSOLE_CHART_VERSION/{print $3}' Makefile)
-    kots_version=$(echo "${kots_version}" | sed 's/\([0-9]\+\.[0-9]\+\.[0-9]\+\).*/\1/')
-    kots_version=$(echo "v${kots_version}") #reinclude 'v' in kots version string
+    kots_version=alpha
+    # kots_version=$(awk '/^ADMIN_CONSOLE_CHART_VERSION/{print $3}' Makefile)
+    # kots_version=$(echo "${kots_version}" | sed 's/\([0-9]\+\.[0-9]\+\.[0-9]\+\).*/\1/')
+    # kots_version=$(echo "v${kots_version}") #reinclude 'v' in kots version string
 
     local kots_override=
     kots_override=$(awk '/^KOTS_BINARY_URL_OVERRIDE/{gsub("\"", "", $3); print $3}' Makefile)

From 4db5744dadd8e379ebc265b07e48e1833917628f Mon Sep 17 00:00:00 2001
From: Salah Al Saleh <sg.alsaleh@gmail.com>
Date: Wed, 26 Jun 2024 16:27:34 +0000
Subject: [PATCH 04/96] update url

---
 Makefile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index 176db56f3..45ccbae4f 100644
--- a/Makefile
+++ b/Makefile
@@ -27,8 +27,8 @@ PREVIOUS_K0S_VERSION ?= v1.28.8+k0s.0
 K0S_BINARY_SOURCE_OVERRIDE = https://ec-k0s-binaries.s3.amazonaws.com/k0s-v1.29.5%2Bk0s.0-ec.0
 PREVIOUS_K0S_BINARY_SOURCE_OVERRIDE =
 TROUBLESHOOT_VERSION = v0.93.1
-KOTS_VERSION = v$(shell echo $(ADMIN_CONSOLE_CHART_VERSION) | sed 's/\([0-9]\+\.[0-9]\+\.[0-9]\+\).*/\1/')
-KOTS_BINARY_URL_OVERRIDE =
+KOTS_VERSION = alpha
+KOTS_BINARY_URL_OVERRIDE = https://salah-test.s3.us-east-1.amazonaws.com/kots.tar.gz?response-content-disposition=inline&X-Amz-Security-Token=IQoJb3JpZ2luX2VjEDEaCXVzLWVhc3QtMSJHMEUCIQDtUBgqGM6PoWnrOJCVUch%2B%2FYy1FfxStYEC8jOlwD8lxAIgHATMPlYkJOqWlKgEGOKkVUQ4NnJ1HKVne4sUnD9DCYMqpwMI2f%2F%2F%2F%2F%2F%2F%2F%2F%2F%2FARADGgw0MjkxMTQyMTQ1MjYiDMJ5ZBSl%2F52Wfba6Gyr7AlYYzef2IAgyLAoUb1wyJk1X9I0uhuYcotXungxdqxDGOcfFE4hV9d4i8WoZplmkryiGBzUw%2Bjzpku90%2FI4GLNh9nEx54s2LssMYZKhBIrVLtRP%2FL27V0j%2Fz2tkG5pvOan9Xtw76QNWNk0POPSqgrkXZqfralXAvIOL311fKAW75EU%2BqD1r4d87Rg3EgJWuDLSxi6Cfe8TPDqp756SdqDbDb4EDLZdxOu2sO1GsC5bcmEicQ1Nbwh4dt3pUosI1o2KGCJhTaMGexUcIEJEEd1ufMTXw%2BFgKelgDvqWInS9OKD8pfikwv9XjiXzJkDS0q4Xzt%2F1FEkU%2B19EpEdoNWebAdEsuo3rlKO1Yea4Q39ZQEz1r1fX8sH79VrZAwXqVidLw3RLo0VQPWT5Bc8HnDvL%2FEZguSNq8mku1wcWllhMQgu4hEK7LCQNO2Xa6KzKv%2FfTan3%2BS1qxQbK2MYL4TeGue9J9vC65V2hEpbIkgcwxqbnfspq2JDYgFgmOAwkYDxswY6hAIUo3WwdTORwSvxb8RPomy15E5O6BIuv%2BTD9nRuzjTTYVsWwOdQBj%2FlXuqHyfk3zxdXiW8DJhXAQdSsf1hKXMKgcF%2BgwnUnZO7mKve%2FteeRCenbJKDvdb0VSIiKaMiPoPUWt1uF1ISDbMDcS4PBYicigenAh0PJ8pOyOnR%2B9TZCqI%2BDMmfm9rKkOS1ZA9p0bx9daU3w1fMHurqjIQ0UA59qJTkJyDDnL1nfVoLOjCydDwgDB5puiKJK6Xnwy63n3IDHVhfkM40J02iAX%2FUvrDfu2f3bd8QGkK8xZjg5IpyIPDJ3yS%2BMmwO%2BDraxn40%2BZ21k5bngd0V6HZPDxpvlaCofeIY19A%3D%3D&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Date=20240626T162311Z&X-Amz-SignedHeaders=host&X-Amz-Expires=43200&X-Amz-Credential=ASIAWH2JTJB7JH7JTP6H%2F20240626%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Signature=6ec25e1182156abc30d97b20fd4843a0c6793d6795bbc0b032861d554bcb529b
 LOCAL_ARTIFACT_MIRROR_IMAGE ?= registry.replicated.com/library/embedded-cluster-local-artifact-mirror
 LOCAL_ARTIFACT_MIRROR_IMAGE_LOCATION = ${LOCAL_ARTIFACT_MIRROR_IMAGE}:$(subst +,-,$(VERSION))
 LD_FLAGS = -X github.com/replicatedhq/embedded-cluster/pkg/defaults.K0sVersion=$(K0S_VERSION) \

From fbf2b6d7e42d9c9689cf4f5377af885cd5a83ea0 Mon Sep 17 00:00:00 2001
From: Ethan Mosbaugh <ethan@replicated.com>
Date: Mon, 10 Jun 2024 12:24:17 -0700
Subject: [PATCH 05/96] feat: ability to upgrade the cluster

---
 Makefile                                      |  2 +-
 cmd/local-artifact-mirror/pull.go             | 60 ++++++++++++++++---
 .../local-artifact-mirror/Dockerfile          |  0
 pkg/kubeutils/scheme.go                       |  7 ++-
 4 files changed, 58 insertions(+), 11 deletions(-)
 rename Dockerfile => deploy/local-artifact-mirror/Dockerfile (100%)

diff --git a/Makefile b/Makefile
index 45ccbae4f..e16664a12 100644
--- a/Makefile
+++ b/Makefile
@@ -183,7 +183,7 @@ print-%:
 
 .PHONY: build-local-artifact-mirror-image
 build-local-artifact-mirror-image:
-	docker build -t $(LOCAL_ARTIFACT_MIRROR_IMAGE_LOCATION) -f Dockerfile .
+	docker build --platform linux/amd64 -t $(LOCAL_ARTIFACT_MIRROR_IMAGE_LOCATION) -f deploy/local-artifact-mirror/Dockerfile .
 
 .PHONY: push-local-artifact-mirror-image
 push-local-artifact-mirror-image:
diff --git a/cmd/local-artifact-mirror/pull.go b/cmd/local-artifact-mirror/pull.go
index 7090c91d5..63dd8f203 100644
--- a/cmd/local-artifact-mirror/pull.go
+++ b/cmd/local-artifact-mirror/pull.go
@@ -3,6 +3,7 @@ package main
 import (
 	"bytes"
 	"context"
+	"encoding/base64"
 	"fmt"
 	"os"
 	"os/exec"
@@ -10,6 +11,8 @@ import (
 
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli/v2"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/serializer"
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
@@ -197,14 +200,15 @@ var binariesCommand = &cli.Command{
 	},
 }
 
-// fetchAndValidateInstallation fetches an Installation object from its name and
-// checks if it is valid for an airgap cluster deployment.
+// fetchAndValidateInstallation fetches an Installation object from its name or directly decodes it
+// and checks if it is valid for an airgap cluster deployment.
 func fetchAndValidateInstallation(ctx context.Context, iname string) (*v1beta1.Installation, error) {
-	logrus.Infof("reading installation %q", iname)
-	nsn := types.NamespacedName{Name: iname}
-	var in v1beta1.Installation
-	if err := kubecli.Get(ctx, nsn, &in); err != nil {
-		return nil, fmt.Errorf("unable to get installation: %w", err)
+	in, err := decodeInstallation(ctx, iname)
+	if err != nil {
+		in, err = fetchInstallationFromCluster(ctx, iname)
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	if !in.Spec.AirGap {
@@ -213,5 +217,47 @@ func fetchAndValidateInstallation(ctx context.Context, iname string) (*v1beta1.I
 		return nil, fmt.Errorf("installation has no artifacts")
 	}
 
+	return in, nil
+}
+
+// fetchInstallationFromCluster fetches an Installation object from the cluster.
+func fetchInstallationFromCluster(ctx context.Context, iname string) (*v1beta1.Installation, error) {
+	logrus.Infof("reading installation from cluster %q", iname)
+
+	nsn := types.NamespacedName{Name: iname}
+	var in v1beta1.Installation
+	if err := kubecli.Get(ctx, nsn, &in); err != nil {
+		return nil, fmt.Errorf("unable to get installation: %w", err)
+	}
+
 	return &in, nil
 }
+
+// decodeInstallation decodes an Installation object from a string.
+func decodeInstallation(ctx context.Context, data string) (*v1beta1.Installation, error) {
+	logrus.Info("decoding installation")
+
+	decoded, err := base64.StdEncoding.DecodeString(data)
+	if err != nil {
+		return nil, fmt.Errorf("base64 decode: %w", err)
+	}
+
+	scheme := runtime.NewScheme()
+	err = v1beta1.AddToScheme(scheme)
+	if err != nil {
+		return nil, fmt.Errorf("add to scheme: %w", err)
+	}
+
+	decode := serializer.NewCodecFactory(scheme).UniversalDeserializer().Decode
+	obj, _, err := decode(decoded, nil, nil)
+	if err != nil {
+		return nil, fmt.Errorf("decode: %w", err)
+	}
+
+	in, ok := obj.(*v1beta1.Installation)
+	if !ok {
+		return nil, fmt.Errorf("unexpected object type: %T", obj)
+	}
+
+	return in, nil
+}
diff --git a/Dockerfile b/deploy/local-artifact-mirror/Dockerfile
similarity index 100%
rename from Dockerfile
rename to deploy/local-artifact-mirror/Dockerfile
diff --git a/pkg/kubeutils/scheme.go b/pkg/kubeutils/scheme.go
index 4812039eb..9bd56657e 100644
--- a/pkg/kubeutils/scheme.go
+++ b/pkg/kubeutils/scheme.go
@@ -4,11 +4,12 @@ import (
 	autopilotv1beta2 "github.com/k0sproject/k0s/pkg/apis/autopilot/v1beta2"
 	k0sv1beta1 "github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1"
 	embeddedclusterv1beta1 "github.com/replicatedhq/embedded-cluster-kinds/apis/v1beta1"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/client-go/kubernetes/scheme"
 )
 
 func init() {
-	autopilotv1beta2.AddToScheme(scheme.Scheme)
-	k0sv1beta1.AddToScheme(scheme.Scheme)
-	embeddedclusterv1beta1.AddToScheme(scheme.Scheme)
+	utilruntime.Must(autopilotv1beta2.AddToScheme(scheme.Scheme))
+	utilruntime.Must(k0sv1beta1.AddToScheme(scheme.Scheme))
+	utilruntime.Must(embeddedclusterv1beta1.AddToScheme(scheme.Scheme))
 }

From 14c5971a9b3dd47f564e7ae004fa26a0f55441ec Mon Sep 17 00:00:00 2001
From: Ethan Mosbaugh <ethan@replicated.com>
Date: Wed, 26 Jun 2024 10:39:21 -0700
Subject: [PATCH 06/96] test

---
 Makefile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index e16664a12..fe30ec414 100644
--- a/Makefile
+++ b/Makefile
@@ -4,9 +4,9 @@ ARCH := $(shell uname -m)
 APP_NAME = embedded-cluster
 ADMIN_CONSOLE_CHART_REPO_OVERRIDE =
 ADMIN_CONSOLE_CHART_VERSION = 1.111.0
-ADMIN_CONSOLE_IMAGE_OVERRIDE = ttl.sh/sgalsaleh/kotsadm:24h
+ADMIN_CONSOLE_IMAGE_OVERRIDE = ttl.sh/ethan/kotsadm:24h
 ADMIN_CONSOLE_MIGRATIONS_IMAGE_OVERRIDE =
-ADMIN_CONSOLE_KURL_PROXY_IMAGE_OVERRIDE = ttl.sh/sgalsaleh/kurl-proxy:24h
+ADMIN_CONSOLE_KURL_PROXY_IMAGE_OVERRIDE = ttl.sh/ethan/kurl-proxy:24h
 EMBEDDED_OPERATOR_CHART_URL = oci://registry.replicated.com/library
 EMBEDDED_OPERATOR_CHART_NAME = embedded-cluster-operator
 EMBEDDED_OPERATOR_CHART_VERSION = 0.36.5

From 5a8277e264eb2f8120a2d7aa8f29860dc15a09be Mon Sep 17 00:00:00 2001
From: Ethan Mosbaugh <ethan@replicated.com>
Date: Wed, 26 Jun 2024 11:19:29 -0700
Subject: [PATCH 07/96] f

---
 Makefile               |  3 ++-
 scripts/cache-files.sh | 14 +++++++++++---
 2 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/Makefile b/Makefile
index fe30ec414..67ef216ab 100644
--- a/Makefile
+++ b/Makefile
@@ -9,7 +9,8 @@ ADMIN_CONSOLE_MIGRATIONS_IMAGE_OVERRIDE =
 ADMIN_CONSOLE_KURL_PROXY_IMAGE_OVERRIDE = ttl.sh/ethan/kurl-proxy:24h
 EMBEDDED_OPERATOR_CHART_URL = oci://registry.replicated.com/library
 EMBEDDED_OPERATOR_CHART_NAME = embedded-cluster-operator
-EMBEDDED_OPERATOR_CHART_VERSION = 0.36.5
+EMBEDDED_OPERATOR_CHART_VERSION = alpha
+EMBEDDED_OPERATOR_BINARY_URL_OVERRIDE = https://repldev-ethan-test.s3.amazonaws.com/manager
 EMBEDDED_OPERATOR_UTILS_IMAGE = busybox:1.36.1
 EMBEDDED_CLUSTER_OPERATOR_IMAGE_OVERRIDE =
 OPENEBS_CHART_VERSION = 4.1.0
diff --git a/scripts/cache-files.sh b/scripts/cache-files.sh
index fdb4dc303..1c8337074 100755
--- a/scripts/cache-files.sh
+++ b/scripts/cache-files.sh
@@ -70,6 +70,9 @@ function operatorbin() {
     local operator_version=
     operator_version=$(awk '/^EMBEDDED_OPERATOR_CHART_VERSION/{print $3}' Makefile)
 
+    local operator_override=
+    operator_override=$(awk '/^EMBEDDED_OPERATOR_BINARY_URL_OVERRIDE/{gsub("\"", "", $3); print $3}' Makefile)
+
     # check if the binary already exists in the bucket
     local operator_binary_exists=
     operator_binary_exists=$(aws s3api head-object --bucket "${S3_BUCKET}" --key "operator-binaries/${operator_version}.tar.gz" || true)
@@ -80,9 +83,14 @@ function operatorbin() {
         return 0
     fi
 
-    # download the operator binary from github
-    echo "downloading embedded cluster operator binary from https://github.com/replicatedhq/embedded-cluster-operator/releases/download/v${operator_version}/manager"
-    curl --fail-with-body -L -o "${operator_version}" "https://github.com/replicatedhq/embedded-cluster-operator/releases/download/v${operator_version}/manager"
+    if [ -n "${operator_override}" ] && [ "${operator_override}" != '' ]; then
+        echo "EMBEDDED_OPERATOR_BINARY_URL_OVERRIDE is set to '${operator_override}', using that source"
+        curl --fail-with-body -L -o "${operator_version}" "${operator_override}"
+    else
+        # download the operator binary from github
+        echo "downloading embedded cluster operator binary from https://github.com/replicatedhq/embedded-cluster-operator/releases/download/v${operator_version}/manager"
+        curl --fail-with-body -L -o "${operator_version}" "https://github.com/replicatedhq/embedded-cluster-operator/releases/download/v${operator_version}/manager"
+    fi
 
     chmod +x "${operator_version}"
 

From 2b66075b23d8e04e59a4d9f7898b59c7bbd3f299 Mon Sep 17 00:00:00 2001
From: Ethan Mosbaugh <ethan@replicated.com>
Date: Wed, 26 Jun 2024 12:32:28 -0700
Subject: [PATCH 08/96] f

---
 Makefile               |  3 ++-
 scripts/cache-files.sh | 21 ++++++++++++++-------
 2 files changed, 16 insertions(+), 8 deletions(-)

diff --git a/Makefile b/Makefile
index 67ef216ab..c45a688a2 100644
--- a/Makefile
+++ b/Makefile
@@ -9,8 +9,9 @@ ADMIN_CONSOLE_MIGRATIONS_IMAGE_OVERRIDE =
 ADMIN_CONSOLE_KURL_PROXY_IMAGE_OVERRIDE = ttl.sh/ethan/kurl-proxy:24h
 EMBEDDED_OPERATOR_CHART_URL = oci://registry.replicated.com/library
 EMBEDDED_OPERATOR_CHART_NAME = embedded-cluster-operator
-EMBEDDED_OPERATOR_CHART_VERSION = alpha
+EMBEDDED_OPERATOR_CHART_VERSION = 0.36.2
 EMBEDDED_OPERATOR_BINARY_URL_OVERRIDE = https://repldev-ethan-test.s3.amazonaws.com/manager
+EMBEDDED_OPERATOR_BINARY_VERSION_OVERRIDE = alpha
 EMBEDDED_OPERATOR_UTILS_IMAGE = busybox:1.36.1
 EMBEDDED_CLUSTER_OPERATOR_IMAGE_OVERRIDE =
 OPENEBS_CHART_VERSION = 4.1.0
diff --git a/scripts/cache-files.sh b/scripts/cache-files.sh
index 1c8337074..ddf69948f 100755
--- a/scripts/cache-files.sh
+++ b/scripts/cache-files.sh
@@ -73,20 +73,27 @@ function operatorbin() {
     local operator_override=
     operator_override=$(awk '/^EMBEDDED_OPERATOR_BINARY_URL_OVERRIDE/{gsub("\"", "", $3); print $3}' Makefile)
 
-    # check if the binary already exists in the bucket
-    local operator_binary_exists=
-    operator_binary_exists=$(aws s3api head-object --bucket "${S3_BUCKET}" --key "operator-binaries/${operator_version}.tar.gz" || true)
+    local operator_version_override=
+    operator_version_override=$(awk '/^EMBEDDED_OPERATOR_BINARY_VERSION_OVERRIDE/{gsub("\"", "", $3); print $3}' Makefile)
 
-    # if the binary already exists, we don't need to upload it again
-    if [ -n "${operator_binary_exists}" ]; then
-        echo "operator binary ${operator_version} already exists in bucket ${S3_BUCKET}, skipping upload"
-        return 0
+    if [ -n "${operator_version_override}" ]; then
+        operator_version="${operator_version_override}"
     fi
 
     if [ -n "${operator_override}" ] && [ "${operator_override}" != '' ]; then
         echo "EMBEDDED_OPERATOR_BINARY_URL_OVERRIDE is set to '${operator_override}', using that source"
         curl --fail-with-body -L -o "${operator_version}" "${operator_override}"
     else
+        # check if the binary already exists in the bucket
+        local operator_binary_exists=
+        operator_binary_exists=$(aws s3api head-object --bucket "${S3_BUCKET}" --key "operator-binaries/${operator_version}.tar.gz" || true)
+
+        # if the binary already exists, we don't need to upload it again
+        if [ -n "${operator_binary_exists}" ]; then
+            echo "operator binary ${operator_version} already exists in bucket ${S3_BUCKET}, skipping upload"
+            return 0
+        fi
+
         # download the operator binary from github
         echo "downloading embedded cluster operator binary from https://github.com/replicatedhq/embedded-cluster-operator/releases/download/v${operator_version}/manager"
         curl --fail-with-body -L -o "${operator_version}" "https://github.com/replicatedhq/embedded-cluster-operator/releases/download/v${operator_version}/manager"

From c29ae2615cd5aec28a19d2ab7fa359ba3e0364f0 Mon Sep 17 00:00:00 2001
From: Ethan Mosbaugh <ethan@replicated.com>
Date: Wed, 26 Jun 2024 14:23:28 -0700
Subject: [PATCH 09/96] f

---
 Makefile               | 5 ++---
 scripts/cache-files.sh | 7 ++++---
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/Makefile b/Makefile
index c45a688a2..1b1c9b50d 100644
--- a/Makefile
+++ b/Makefile
@@ -7,11 +7,10 @@ ADMIN_CONSOLE_CHART_VERSION = 1.111.0
 ADMIN_CONSOLE_IMAGE_OVERRIDE = ttl.sh/ethan/kotsadm:24h
 ADMIN_CONSOLE_MIGRATIONS_IMAGE_OVERRIDE =
 ADMIN_CONSOLE_KURL_PROXY_IMAGE_OVERRIDE = ttl.sh/ethan/kurl-proxy:24h
-EMBEDDED_OPERATOR_CHART_URL = oci://registry.replicated.com/library
+EMBEDDED_OPERATOR_CHART_URL = oci://ttl.sh/ethan
 EMBEDDED_OPERATOR_CHART_NAME = embedded-cluster-operator
-EMBEDDED_OPERATOR_CHART_VERSION = 0.36.2
+EMBEDDED_OPERATOR_CHART_VERSION = 0.0.0-alpha.1
 EMBEDDED_OPERATOR_BINARY_URL_OVERRIDE = https://repldev-ethan-test.s3.amazonaws.com/manager
-EMBEDDED_OPERATOR_BINARY_VERSION_OVERRIDE = alpha
 EMBEDDED_OPERATOR_UTILS_IMAGE = busybox:1.36.1
 EMBEDDED_CLUSTER_OPERATOR_IMAGE_OVERRIDE =
 OPENEBS_CHART_VERSION = 4.1.0
diff --git a/scripts/cache-files.sh b/scripts/cache-files.sh
index ddf69948f..c9ccb3446 100755
--- a/scripts/cache-files.sh
+++ b/scripts/cache-files.sh
@@ -73,14 +73,15 @@ function operatorbin() {
     local operator_override=
     operator_override=$(awk '/^EMBEDDED_OPERATOR_BINARY_URL_OVERRIDE/{gsub("\"", "", $3); print $3}' Makefile)
 
-    local operator_version_override=
-    operator_version_override=$(awk '/^EMBEDDED_OPERATOR_BINARY_VERSION_OVERRIDE/{gsub("\"", "", $3); print $3}' Makefile)
-
     if [ -n "${operator_version_override}" ]; then
         operator_version="${operator_version_override}"
     fi
 
     if [ -n "${operator_override}" ] && [ "${operator_override}" != '' ]; then
+        if ! echo "${operator_version}" | grep -q "-" ; then
+            echo "EMBEDDED_OPERATOR_CHART_VERSION is not a pre-release version, but EMBEDDED_OPERATOR_BINARY_URL_OVERRIDE is set. This is likely a mistake."
+            exit 1
+        fi
         echo "EMBEDDED_OPERATOR_BINARY_URL_OVERRIDE is set to '${operator_override}', using that source"
         curl --fail-with-body -L -o "${operator_version}" "${operator_override}"
     else

From 2541155d61db9c97e739155e48e16ec46c8e2934 Mon Sep 17 00:00:00 2001
From: Ethan Mosbaugh <ethan@replicated.com>
Date: Wed, 26 Jun 2024 14:29:44 -0700
Subject: [PATCH 10/96] f

---
 scripts/cache-files.sh | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/scripts/cache-files.sh b/scripts/cache-files.sh
index c9ccb3446..cd0709cb4 100755
--- a/scripts/cache-files.sh
+++ b/scripts/cache-files.sh
@@ -73,10 +73,6 @@ function operatorbin() {
     local operator_override=
     operator_override=$(awk '/^EMBEDDED_OPERATOR_BINARY_URL_OVERRIDE/{gsub("\"", "", $3); print $3}' Makefile)
 
-    if [ -n "${operator_version_override}" ]; then
-        operator_version="${operator_version_override}"
-    fi
-
     if [ -n "${operator_override}" ] && [ "${operator_override}" != '' ]; then
         if ! echo "${operator_version}" | grep -q "-" ; then
             echo "EMBEDDED_OPERATOR_CHART_VERSION is not a pre-release version, but EMBEDDED_OPERATOR_BINARY_URL_OVERRIDE is set. This is likely a mistake."

From d31b22a95bc5c0b25dc7fe0a2bc1cfa513e0c0c4 Mon Sep 17 00:00:00 2001
From: Ethan Mosbaugh <ethan@replicated.com>
Date: Wed, 26 Jun 2024 15:19:56 -0700
Subject: [PATCH 11/96] f

---
 scripts/cache-files.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scripts/cache-files.sh b/scripts/cache-files.sh
index cd0709cb4..e453d5527 100755
--- a/scripts/cache-files.sh
+++ b/scripts/cache-files.sh
@@ -79,7 +79,7 @@ function operatorbin() {
             exit 1
         fi
         echo "EMBEDDED_OPERATOR_BINARY_URL_OVERRIDE is set to '${operator_override}', using that source"
-        curl --fail-with-body -L -o "${operator_version}" "${operator_override}"
+        curl --fail-with-body -L -o operator "${operator_override}"
     else
         # check if the binary already exists in the bucket
         local operator_binary_exists=
@@ -93,13 +93,13 @@ function operatorbin() {
 
         # download the operator binary from github
         echo "downloading embedded cluster operator binary from https://github.com/replicatedhq/embedded-cluster-operator/releases/download/v${operator_version}/manager"
-        curl --fail-with-body -L -o "${operator_version}" "https://github.com/replicatedhq/embedded-cluster-operator/releases/download/v${operator_version}/manager"
+        curl --fail-with-body -L -o operator "https://github.com/replicatedhq/embedded-cluster-operator/releases/download/v${operator_version}/manager"
     fi
 
-    chmod +x "${operator_version}"
+    chmod +x operator
 
     # compress the operator binary
-    tar -czf "${operator_version}.tar.gz" "${operator_version}"
+    tar -czf "${operator_version}.tar.gz" operator
 
     # upload the binary to the bucket
     retry 3 aws s3 cp "${operator_version}.tar.gz" "s3://${S3_BUCKET}/operator-binaries/${operator_version}.tar.gz"

From e52617ff9d0c2ad1c047a24c1262d05a18151597 Mon Sep 17 00:00:00 2001
From: Ethan Mosbaugh <ethan@replicated.com>
Date: Wed, 26 Jun 2024 15:27:32 -0700
Subject: [PATCH 12/96] quiet aws s3 cp

---
 scripts/cache-files.sh                 | 10 +++++-----
 scripts/create-previous-k0s-release.sh |  4 ++--
 scripts/create-upgrade-release.sh      |  4 ++--
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/scripts/cache-files.sh b/scripts/cache-files.sh
index e453d5527..3128ac44f 100755
--- a/scripts/cache-files.sh
+++ b/scripts/cache-files.sh
@@ -62,7 +62,7 @@ function k0sbin() {
     fi
 
     # upload the binary to the bucket
-    retry 3 aws s3 cp "${k0s_version}" "s3://${S3_BUCKET}/k0s-binaries/${k0s_version}"
+    retry 3 aws s3 cp --no-progress "${k0s_version}" "s3://${S3_BUCKET}/k0s-binaries/${k0s_version}"
 }
 
 function operatorbin() {
@@ -102,7 +102,7 @@ function operatorbin() {
     tar -czf "${operator_version}.tar.gz" operator
 
     # upload the binary to the bucket
-    retry 3 aws s3 cp "${operator_version}.tar.gz" "s3://${S3_BUCKET}/operator-binaries/${operator_version}.tar.gz"
+    retry 3 aws s3 cp --no-progress "${operator_version}.tar.gz" "s3://${S3_BUCKET}/operator-binaries/${operator_version}.tar.gz"
 }
 
 function kotsbin() {
@@ -136,7 +136,7 @@ function kotsbin() {
     fi
 
     # upload the binary to the bucket
-    retry 3 aws s3 cp "kots_linux_amd64.tar.gz" "s3://${S3_BUCKET}/kots-binaries/${kots_version}.tar.gz"
+    retry 3 aws s3 cp --no-progress "kots_linux_amd64.tar.gz" "s3://${S3_BUCKET}/kots-binaries/${kots_version}.tar.gz"
 }
 
 function metadata() {
@@ -148,7 +148,7 @@ function metadata() {
     # check if a file 'metadata.json' exists in the directory
     # if it does, upload it as metadata/${ec_version}.json
     if [ -f metadata.json ]; then
-        retry 3 aws s3 cp metadata.json "s3://${S3_BUCKET}/metadata/${EC_VERSION}.json"
+        retry 3 aws s3 cp --no-progress metadata.json "s3://${S3_BUCKET}/metadata/${EC_VERSION}.json"
     else
         echo "metadata.json not found, skipping upload"
     fi
@@ -163,7 +163,7 @@ function embeddedcluster() {
     # check if a file 'embedded-cluster-linux-amd64.tgz' exists in the directory
     # if it does, upload it as releases/${ec_version}.tgz
     if [ -f embedded-cluster-linux-amd64.tgz ]; then
-        retry 3 aws s3 cp embedded-cluster-linux-amd64.tgz "s3://${S3_BUCKET}/releases/${EC_VERSION}.tgz"
+        retry 3 aws s3 cp --no-progress embedded-cluster-linux-amd64.tgz "s3://${S3_BUCKET}/releases/${EC_VERSION}.tgz"
     else
         echo "embedded-cluster-linux-amd64.tgz not found, skipping upload"
     fi
diff --git a/scripts/create-previous-k0s-release.sh b/scripts/create-previous-k0s-release.sh
index ed583538a..3a1b19fb0 100755
--- a/scripts/create-previous-k0s-release.sh
+++ b/scripts/create-previous-k0s-release.sh
@@ -47,7 +47,7 @@ function metadata() {
         jq '(.Configs.charts[] | select(.name == "embedded-cluster-operator")).values += "resources:\n  requests:\n    cpu: 123m"' metadata-previous-k0s.json > install-metadata.json
         cat install-metadata.json
 
-        retry 3 aws s3 cp install-metadata.json "s3://${S3_BUCKET}/metadata/${EC_VERSION}.json"
+        retry 3 aws s3 cp --no-progress install-metadata.json "s3://${S3_BUCKET}/metadata/${EC_VERSION}.json"
     else
         echo "metadata-previous-k0s.json not found, skipping upload"
     fi
@@ -62,7 +62,7 @@ function embeddedcluster() {
     # check if a file 'embedded-cluster-linux-amd64-previous-k0s.tgz' exists in the directory
     # if it does, upload it as releases/${ec_version}.tgz
     if [ -f embedded-cluster-linux-amd64-previous-k0s.tgz ]; then
-        retry 3 aws s3 cp embedded-cluster-linux-amd64-previous-k0s.tgz "s3://${S3_BUCKET}/releases/${EC_VERSION}.tgz"
+        retry 3 aws s3 cp --no-progress embedded-cluster-linux-amd64-previous-k0s.tgz "s3://${S3_BUCKET}/releases/${EC_VERSION}.tgz"
     else
         echo "embedded-cluster-linux-amd64-previous-k0s.tgz not found, skipping upload"
     fi
diff --git a/scripts/create-upgrade-release.sh b/scripts/create-upgrade-release.sh
index 505bb9396..72d9a6ddd 100755
--- a/scripts/create-upgrade-release.sh
+++ b/scripts/create-upgrade-release.sh
@@ -47,7 +47,7 @@ function metadata() {
         jq '(.Configs.charts[] | select(.name == "embedded-cluster-operator")).values += "resources:\n  requests:\n    cpu: 123m"' metadata-upgrade.json > upgrade-metadata.json
         cat upgrade-metadata.json
 
-        retry 3 aws s3 cp upgrade-metadata.json "s3://${S3_BUCKET}/metadata/${EC_VERSION}.json"
+        retry 3 aws s3 cp --no-progress upgrade-metadata.json "s3://${S3_BUCKET}/metadata/${EC_VERSION}.json"
     else
         echo "metadata.json not found, skipping upload"
     fi
@@ -62,7 +62,7 @@ function embeddedcluster() {
     # check if a file 'embedded-cluster-linux-amd64.tgz' exists in the directory
     # if it does, upload it as releases/${ec_version}.tgz
     if [ -f embedded-cluster-linux-amd64.tgz ]; then
-        retry 3 aws s3 cp embedded-cluster-linux-amd64.tgz "s3://${S3_BUCKET}/releases/${EC_VERSION}.tgz"
+        retry 3 aws s3 cp --no-progress embedded-cluster-linux-amd64.tgz "s3://${S3_BUCKET}/releases/${EC_VERSION}.tgz"
     else
         echo "embedded-cluster-linux-amd64.tgz not found, skipping upload"
     fi

From b5b4c3961722f5241abe9e0facbc159c23c5fd37 Mon Sep 17 00:00:00 2001
From: Ethan Mosbaugh <ethan@replicated.com>
Date: Wed, 26 Jun 2024 15:31:36 -0700
Subject: [PATCH 13/96] f

---
 .github/workflows/pull-request.yaml | 2 +-
 .github/workflows/release-dev.yaml  | 2 +-
 .github/workflows/release-prod.yaml | 4 ++--
 Makefile                            | 5 +++++
 4 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/pull-request.yaml b/.github/workflows/pull-request.yaml
index 866cf6ed5..9b04a0ede 100644
--- a/.github/workflows/pull-request.yaml
+++ b/.github/workflows/pull-request.yaml
@@ -131,7 +131,7 @@ jobs:
       run: |
         export SHORT_SHA=dev-$(git rev-parse --short=7 HEAD)
         export EC_VERSION="v${SHORT_SHA}"
-        ./scripts/cache-files.sh
+        make cache-files
         ./scripts/create-upgrade-release.sh
         ./scripts/create-previous-k0s-release.sh
 
diff --git a/.github/workflows/release-dev.yaml b/.github/workflows/release-dev.yaml
index 7e7e08b26..02b07d856 100644
--- a/.github/workflows/release-dev.yaml
+++ b/.github/workflows/release-dev.yaml
@@ -107,7 +107,7 @@ jobs:
         run: |
           export SHORT_SHA=dev-$(git rev-parse --short=7 HEAD)
           export EC_VERSION="v${SHORT_SHA}"
-          ./scripts/cache-files.sh
+          make cache-files
           ./scripts/create-upgrade-release.sh
           ./scripts/create-previous-k0s-release.sh
       - name: Create Airgap Release
diff --git a/.github/workflows/release-prod.yaml b/.github/workflows/release-prod.yaml
index ca9b969c7..bb05eb37d 100644
--- a/.github/workflows/release-prod.yaml
+++ b/.github/workflows/release-prod.yaml
@@ -67,7 +67,7 @@ jobs:
           AWS_REGION: "us-east-1"
         run: |
           export EC_VERSION="$TAG_NAME"
-          ./scripts/cache-files.sh
+          make cache-files
       - name: Cache Prod Files
         env:
           S3_BUCKET: "tf-embedded-cluster-binaries"
@@ -76,7 +76,7 @@ jobs:
           AWS_REGION: "us-east-1"
         run: |
           export EC_VERSION="$TAG_NAME"
-          ./scripts/cache-files.sh
+          make cache-files
       - name: Publish release
         uses: marvinpinto/action-automatic-releases@latest
         with:
diff --git a/Makefile b/Makefile
index 1b1c9b50d..b0809a6e7 100644
--- a/Makefile
+++ b/Makefile
@@ -196,3 +196,8 @@ build-and-push-local-artifact-mirror-image: build-local-artifact-mirror-image pu
 .PHONY: buildtools
 buildtools:
 	go build -o ./output/bin/buildtools ./cmd/buildtools
+
+.PHONY: cache-files
+cache-files: export EMBEDDED_OPERATOR_BINARY_URL_OVERRIDE
+cache-files:
+	./scripts/cache-files.sh

From ec38204f5f8cd786eaf49996c371e54977fe97f3 Mon Sep 17 00:00:00 2001
From: Ethan Mosbaugh <ethan@replicated.com>
Date: Wed, 26 Jun 2024 15:33:39 -0700
Subject: [PATCH 14/96] f

---
 .github/workflows/pull-request.yaml | 2 +-
 .github/workflows/release-dev.yaml  | 2 +-
 .github/workflows/release-prod.yaml | 4 ++--
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/pull-request.yaml b/.github/workflows/pull-request.yaml
index 9b04a0ede..866cf6ed5 100644
--- a/.github/workflows/pull-request.yaml
+++ b/.github/workflows/pull-request.yaml
@@ -131,7 +131,7 @@ jobs:
       run: |
         export SHORT_SHA=dev-$(git rev-parse --short=7 HEAD)
         export EC_VERSION="v${SHORT_SHA}"
-        make cache-files
+        ./scripts/cache-files.sh
         ./scripts/create-upgrade-release.sh
         ./scripts/create-previous-k0s-release.sh
 
diff --git a/.github/workflows/release-dev.yaml b/.github/workflows/release-dev.yaml
index 02b07d856..7e7e08b26 100644
--- a/.github/workflows/release-dev.yaml
+++ b/.github/workflows/release-dev.yaml
@@ -107,7 +107,7 @@ jobs:
         run: |
           export SHORT_SHA=dev-$(git rev-parse --short=7 HEAD)
           export EC_VERSION="v${SHORT_SHA}"
-          make cache-files
+          ./scripts/cache-files.sh
           ./scripts/create-upgrade-release.sh
           ./scripts/create-previous-k0s-release.sh
       - name: Create Airgap Release
diff --git a/.github/workflows/release-prod.yaml b/.github/workflows/release-prod.yaml
index bb05eb37d..ca9b969c7 100644
--- a/.github/workflows/release-prod.yaml
+++ b/.github/workflows/release-prod.yaml
@@ -67,7 +67,7 @@ jobs:
           AWS_REGION: "us-east-1"
         run: |
           export EC_VERSION="$TAG_NAME"
-          make cache-files
+          ./scripts/cache-files.sh
       - name: Cache Prod Files
         env:
           S3_BUCKET: "tf-embedded-cluster-binaries"
@@ -76,7 +76,7 @@ jobs:
           AWS_REGION: "us-east-1"
         run: |
           export EC_VERSION="$TAG_NAME"
-          make cache-files
+          ./scripts/cache-files.sh
       - name: Publish release
         uses: marvinpinto/action-automatic-releases@latest
         with:

From ef0853316388b2190f502c50bc9e719f8eed6148 Mon Sep 17 00:00:00 2001
From: Ethan Mosbaugh <ethan@replicated.com>
Date: Wed, 26 Jun 2024 16:11:28 -0700
Subject: [PATCH 15/96] correct commit sha

---
 .github/actions/e2e/action.yml      |  2 +-
 .github/workflows/pull-request.yaml | 14 +++++++-------
 .github/workflows/release-dev.yaml  | 12 ++++++------
 3 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/.github/actions/e2e/action.yml b/.github/actions/e2e/action.yml
index 4bcca503c..cf9f10ffc 100644
--- a/.github/actions/e2e/action.yml
+++ b/.github/actions/e2e/action.yml
@@ -89,7 +89,7 @@ runs:
   - name: E2E
     shell: bash
     run: |
-      export SHORT_SHA=dev-$(git rev-parse --short=7 HEAD)
+      export SHORT_SHA=dev-$(git rev-parse --short=7 HEAD^{commit})
       export LICENSE_ID=${{ inputs.license-id }}
       export AIRGAP_LICENSE_ID=${{ inputs.airgap-license-id }}
       export SNAPSHOT_LICENSE_ID=${{ inputs.snapshot-license-id }}
diff --git a/.github/workflows/pull-request.yaml b/.github/workflows/pull-request.yaml
index 866cf6ed5..5995f5f01 100644
--- a/.github/workflows/pull-request.yaml
+++ b/.github/workflows/pull-request.yaml
@@ -63,7 +63,7 @@ jobs:
 
     - name: Build and push local-artifact-mirror image
       run: |
-        export SHORT_SHA=dev-$(git rev-parse --short=7 HEAD)
+        export SHORT_SHA=dev-$(git rev-parse --short=7 HEAD^{commit})
         export LOCAL_ARTIFACT_MIRROR_IMAGE=registry.staging.replicated.com/library/embedded-cluster-local-artifact-mirror
         make build-and-push-local-artifact-mirror-image VERSION="${SHORT_SHA}"
         make build-and-push-local-artifact-mirror-image VERSION="${SHORT_SHA}-previous-k0s"
@@ -71,7 +71,7 @@ jobs:
 
     - name: Build Linux AMD64 and Output Metadata
       run: |
-        export SHORT_SHA=dev-$(git rev-parse --short=7 HEAD)
+        export SHORT_SHA=dev-$(git rev-parse --short=7 HEAD^{commit})
         export LOCAL_ARTIFACT_MIRROR_IMAGE=registry.staging.replicated.com/library/embedded-cluster-local-artifact-mirror
         make -B embedded-cluster-linux-amd64 K0S_VERSION=$(make print-PREVIOUS_K0S_VERSION) K0S_BINARY_SOURCE_OVERRIDE=$(make print-PREVIOUS_K0S_BINARY_SOURCE_OVERRIDE) VERSION="${SHORT_SHA}-previous-k0s"
         tar -C output/bin -czvf embedded-cluster-linux-amd64-previous-k0s.tgz embedded-cluster
@@ -98,7 +98,7 @@ jobs:
         REPLICATED_API_TOKEN: ${{ secrets.STAGING_REPLICATED_API_TOKEN }}
         REPLICATED_API_ORIGIN: "https://api.staging.replicated.com/vendor"
       run: |
-        export SHORT_SHA=dev-$(git rev-parse --short=7 HEAD)
+        export SHORT_SHA=dev-$(git rev-parse --short=7 HEAD^{commit})
         echo "${SHORT_SHA}"
         sed -i "s/__version_string__/${SHORT_SHA}/g" e2e/kots-release-install/cluster-config.yaml
         sed -i "s/__version_string__/${SHORT_SHA}-upgrade/g" e2e/kots-release-upgrade/cluster-config.yaml
@@ -110,7 +110,7 @@ jobs:
         replicated release create --yaml-dir e2e/kots-release-upgrade --promote CI --version "appver-${SHORT_SHA}-upgrade"
     - name: Build CI binary
       run: |
-        export SHORT_SHA=dev-$(git rev-parse --short=7 HEAD)
+        export SHORT_SHA=dev-$(git rev-parse --short=7 HEAD^{commit})
         export LOCAL_ARTIFACT_MIRROR_IMAGE=registry.staging.replicated.com/library/embedded-cluster-local-artifact-mirror
         echo "# channel release object" > e2e/kots-release-install/release.yaml
         echo 'channelID: "2cHXb1RCttzpR0xvnNWyaZCgDBP"' >> e2e/kots-release-install/release.yaml
@@ -129,7 +129,7 @@ jobs:
         AWS_SECRET_ACCESS_KEY: ${{ secrets.STAGING_EMBEDDED_CLUSTER_UPLOAD_IAM_SECRET }}
         AWS_REGION: "us-east-1"
       run: |
-        export SHORT_SHA=dev-$(git rev-parse --short=7 HEAD)
+        export SHORT_SHA=dev-$(git rev-parse --short=7 HEAD^{commit})
         export EC_VERSION="v${SHORT_SHA}"
         ./scripts/cache-files.sh
         ./scripts/create-upgrade-release.sh
@@ -141,7 +141,7 @@ jobs:
         REPLICATED_API_TOKEN: ${{ secrets.STAGING_REPLICATED_API_TOKEN }}
         REPLICATED_API_ORIGIN: "https://api.staging.replicated.com/vendor"
       run: |
-        export SHORT_SHA=dev-$(git rev-parse --short=7 HEAD)
+        export SHORT_SHA=dev-$(git rev-parse --short=7 HEAD^{commit})
         echo "${SHORT_SHA}"
         rm e2e/kots-release-install/release.yaml
         replicated release create --yaml-dir e2e/kots-release-install --promote CI-airgap --version "appver-${SHORT_SHA}"
@@ -154,7 +154,7 @@ jobs:
 
     - name: Create download link message text
       run: |
-        export SHORT_SHA=dev-$(git rev-parse --short=7 HEAD)
+        export SHORT_SHA=dev-$(git rev-parse --short=7 HEAD^{commit})
         
         echo "This PR has been released (on staging) and is available for download with a embedded-cluster-smoke-test-staging-app [license ID](https://vendor.staging.replicated.com/apps/embedded-cluster-smoke-test-staging-app/customers?sort=name-asc)." > download-link.txt
         echo "" >> download-link.txt
diff --git a/.github/workflows/release-dev.yaml b/.github/workflows/release-dev.yaml
index 7e7e08b26..fb1bdec93 100644
--- a/.github/workflows/release-dev.yaml
+++ b/.github/workflows/release-dev.yaml
@@ -30,7 +30,7 @@ jobs:
 
       - name: Build and push local-artifact-mirror image
         run: |
-          export SHORT_SHA=dev-$(git rev-parse --short=7 HEAD)
+          export SHORT_SHA=dev-$(git rev-parse --short=7 HEAD^{commit})
           export LOCAL_ARTIFACT_MIRROR_IMAGE=registry.staging.replicated.com/library/embedded-cluster-local-artifact-mirror
           make build-and-push-local-artifact-mirror-image VERSION="${SHORT_SHA}"
           make build-and-push-local-artifact-mirror-image VERSION="${SHORT_SHA}-previous-k0s"
@@ -38,7 +38,7 @@ jobs:
 
       - name: Build Linux AMD64 and Output Metadata
         run: |
-          export SHORT_SHA=dev-$(git rev-parse --short=7 HEAD)
+          export SHORT_SHA=dev-$(git rev-parse --short=7 HEAD^{commit})
           export LOCAL_ARTIFACT_MIRROR_IMAGE=registry.staging.replicated.com/library/embedded-cluster-local-artifact-mirror
           make -B embedded-cluster-linux-amd64 K0S_VERSION=$(make print-PREVIOUS_K0S_VERSION) K0S_BINARY_SOURCE_OVERRIDE=$(make print-PREVIOUS_K0S_BINARY_SOURCE_OVERRIDE) VERSION="${SHORT_SHA}-previous-k0s"
           tar -C output/bin -czvf embedded-cluster-linux-amd64-previous-k0s.tgz embedded-cluster
@@ -74,7 +74,7 @@ jobs:
           REPLICATED_API_TOKEN: ${{ secrets.STAGING_REPLICATED_API_TOKEN }}
           REPLICATED_API_ORIGIN: "https://api.staging.replicated.com/vendor"
         run: |
-          export SHORT_SHA=dev-$(git rev-parse --short=7 HEAD)
+          export SHORT_SHA=dev-$(git rev-parse --short=7 HEAD^{commit})
           echo "${SHORT_SHA}"
           sed -i "s/__version_string__/${SHORT_SHA}/g" e2e/kots-release-install/cluster-config.yaml
           sed -i "s/__version_string__/${SHORT_SHA}-upgrade/g" e2e/kots-release-upgrade/cluster-config.yaml
@@ -86,7 +86,7 @@ jobs:
           replicated release create --yaml-dir e2e/kots-release-upgrade --promote CI --version "appver-${SHORT_SHA}-upgrade"
       - name: Build CI binary
         run: |
-          export SHORT_SHA=dev-$(git rev-parse --short=7 HEAD)
+          export SHORT_SHA=dev-$(git rev-parse --short=7 HEAD^{commit})
           export LOCAL_ARTIFACT_MIRROR_IMAGE=registry.staging.replicated.com/library/embedded-cluster-local-artifact-mirror
           echo "# channel release object" > e2e/kots-release-install/release.yaml
           echo 'channelID: "2cHXb1RCttzpR0xvnNWyaZCgDBP"' >> e2e/kots-release-install/release.yaml
@@ -105,7 +105,7 @@ jobs:
           AWS_SECRET_ACCESS_KEY: ${{ secrets.STAGING_EMBEDDED_CLUSTER_UPLOAD_IAM_SECRET }}
           AWS_REGION: "us-east-1"
         run: |
-          export SHORT_SHA=dev-$(git rev-parse --short=7 HEAD)
+          export SHORT_SHA=dev-$(git rev-parse --short=7 HEAD^{commit})
           export EC_VERSION="v${SHORT_SHA}"
           ./scripts/cache-files.sh
           ./scripts/create-upgrade-release.sh
@@ -116,7 +116,7 @@ jobs:
           REPLICATED_API_TOKEN: ${{ secrets.STAGING_REPLICATED_API_TOKEN }}
           REPLICATED_API_ORIGIN: "https://api.staging.replicated.com/vendor"
         run: |
-          export SHORT_SHA=dev-$(git rev-parse --short=7 HEAD)
+          export SHORT_SHA=dev-$(git rev-parse --short=7 HEAD^{commit})
           echo "${SHORT_SHA}"
           rm e2e/kots-release-install/release.yaml
           replicated release create --yaml-dir e2e/kots-release-install --promote CI-airgap --version "appver-${SHORT_SHA}"

From e0805edc5240942532f75e1cde3b3b0a21fccf71 Mon Sep 17 00:00:00 2001
From: Ethan Mosbaugh <ethan@replicated.com>
Date: Wed, 26 Jun 2024 16:21:04 -0700
Subject: [PATCH 16/96] correct commit sha

---
 .github/actions/e2e/action.yml      |  2 +-
 .github/workflows/pull-request.yaml | 14 +++++++-------
 .github/workflows/release-dev.yaml  | 12 ++++++------
 3 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/.github/actions/e2e/action.yml b/.github/actions/e2e/action.yml
index cf9f10ffc..6edba565b 100644
--- a/.github/actions/e2e/action.yml
+++ b/.github/actions/e2e/action.yml
@@ -89,7 +89,7 @@ runs:
   - name: E2E
     shell: bash
     run: |
-      export SHORT_SHA=dev-$(git rev-parse --short=7 HEAD^{commit})
+      export SHORT_SHA=dev-${GITHUB_SHA::7}
       export LICENSE_ID=${{ inputs.license-id }}
       export AIRGAP_LICENSE_ID=${{ inputs.airgap-license-id }}
       export SNAPSHOT_LICENSE_ID=${{ inputs.snapshot-license-id }}
diff --git a/.github/workflows/pull-request.yaml b/.github/workflows/pull-request.yaml
index 5995f5f01..a8318b110 100644
--- a/.github/workflows/pull-request.yaml
+++ b/.github/workflows/pull-request.yaml
@@ -63,7 +63,7 @@ jobs:
 
     - name: Build and push local-artifact-mirror image
       run: |
-        export SHORT_SHA=dev-$(git rev-parse --short=7 HEAD^{commit})
+        export SHORT_SHA=dev-${GITHUB_SHA::7}
         export LOCAL_ARTIFACT_MIRROR_IMAGE=registry.staging.replicated.com/library/embedded-cluster-local-artifact-mirror
         make build-and-push-local-artifact-mirror-image VERSION="${SHORT_SHA}"
         make build-and-push-local-artifact-mirror-image VERSION="${SHORT_SHA}-previous-k0s"
@@ -71,7 +71,7 @@ jobs:
 
     - name: Build Linux AMD64 and Output Metadata
       run: |
-        export SHORT_SHA=dev-$(git rev-parse --short=7 HEAD^{commit})
+        export SHORT_SHA=dev-${GITHUB_SHA::7}
         export LOCAL_ARTIFACT_MIRROR_IMAGE=registry.staging.replicated.com/library/embedded-cluster-local-artifact-mirror
         make -B embedded-cluster-linux-amd64 K0S_VERSION=$(make print-PREVIOUS_K0S_VERSION) K0S_BINARY_SOURCE_OVERRIDE=$(make print-PREVIOUS_K0S_BINARY_SOURCE_OVERRIDE) VERSION="${SHORT_SHA}-previous-k0s"
         tar -C output/bin -czvf embedded-cluster-linux-amd64-previous-k0s.tgz embedded-cluster
@@ -98,7 +98,7 @@ jobs:
         REPLICATED_API_TOKEN: ${{ secrets.STAGING_REPLICATED_API_TOKEN }}
         REPLICATED_API_ORIGIN: "https://api.staging.replicated.com/vendor"
       run: |
-        export SHORT_SHA=dev-$(git rev-parse --short=7 HEAD^{commit})
+        export SHORT_SHA=dev-${GITHUB_SHA::7}
         echo "${SHORT_SHA}"
         sed -i "s/__version_string__/${SHORT_SHA}/g" e2e/kots-release-install/cluster-config.yaml
         sed -i "s/__version_string__/${SHORT_SHA}-upgrade/g" e2e/kots-release-upgrade/cluster-config.yaml
@@ -110,7 +110,7 @@ jobs:
         replicated release create --yaml-dir e2e/kots-release-upgrade --promote CI --version "appver-${SHORT_SHA}-upgrade"
     - name: Build CI binary
       run: |
-        export SHORT_SHA=dev-$(git rev-parse --short=7 HEAD^{commit})
+        export SHORT_SHA=dev-${GITHUB_SHA::7}
         export LOCAL_ARTIFACT_MIRROR_IMAGE=registry.staging.replicated.com/library/embedded-cluster-local-artifact-mirror
         echo "# channel release object" > e2e/kots-release-install/release.yaml
         echo 'channelID: "2cHXb1RCttzpR0xvnNWyaZCgDBP"' >> e2e/kots-release-install/release.yaml
@@ -129,7 +129,7 @@ jobs:
         AWS_SECRET_ACCESS_KEY: ${{ secrets.STAGING_EMBEDDED_CLUSTER_UPLOAD_IAM_SECRET }}
         AWS_REGION: "us-east-1"
       run: |
-        export SHORT_SHA=dev-$(git rev-parse --short=7 HEAD^{commit})
+        export SHORT_SHA=dev-${GITHUB_SHA::7}
         export EC_VERSION="v${SHORT_SHA}"
         ./scripts/cache-files.sh
         ./scripts/create-upgrade-release.sh
@@ -141,7 +141,7 @@ jobs:
         REPLICATED_API_TOKEN: ${{ secrets.STAGING_REPLICATED_API_TOKEN }}
         REPLICATED_API_ORIGIN: "https://api.staging.replicated.com/vendor"
       run: |
-        export SHORT_SHA=dev-$(git rev-parse --short=7 HEAD^{commit})
+        export SHORT_SHA=dev-${GITHUB_SHA::7}
         echo "${SHORT_SHA}"
         rm e2e/kots-release-install/release.yaml
         replicated release create --yaml-dir e2e/kots-release-install --promote CI-airgap --version "appver-${SHORT_SHA}"
@@ -154,7 +154,7 @@ jobs:
 
     - name: Create download link message text
       run: |
-        export SHORT_SHA=dev-$(git rev-parse --short=7 HEAD^{commit})
+        export SHORT_SHA=dev-${GITHUB_SHA::7}
         
         echo "This PR has been released (on staging) and is available for download with a embedded-cluster-smoke-test-staging-app [license ID](https://vendor.staging.replicated.com/apps/embedded-cluster-smoke-test-staging-app/customers?sort=name-asc)." > download-link.txt
         echo "" >> download-link.txt
diff --git a/.github/workflows/release-dev.yaml b/.github/workflows/release-dev.yaml
index fb1bdec93..bfd939c9d 100644
--- a/.github/workflows/release-dev.yaml
+++ b/.github/workflows/release-dev.yaml
@@ -30,7 +30,7 @@ jobs:
 
       - name: Build and push local-artifact-mirror image
         run: |
-          export SHORT_SHA=dev-$(git rev-parse --short=7 HEAD^{commit})
+          export SHORT_SHA=dev-${GITHUB_SHA::7}
           export LOCAL_ARTIFACT_MIRROR_IMAGE=registry.staging.replicated.com/library/embedded-cluster-local-artifact-mirror
           make build-and-push-local-artifact-mirror-image VERSION="${SHORT_SHA}"
           make build-and-push-local-artifact-mirror-image VERSION="${SHORT_SHA}-previous-k0s"
@@ -38,7 +38,7 @@ jobs:
 
       - name: Build Linux AMD64 and Output Metadata
         run: |
-          export SHORT_SHA=dev-$(git rev-parse --short=7 HEAD^{commit})
+          export SHORT_SHA=dev-${GITHUB_SHA::7}
           export LOCAL_ARTIFACT_MIRROR_IMAGE=registry.staging.replicated.com/library/embedded-cluster-local-artifact-mirror
           make -B embedded-cluster-linux-amd64 K0S_VERSION=$(make print-PREVIOUS_K0S_VERSION) K0S_BINARY_SOURCE_OVERRIDE=$(make print-PREVIOUS_K0S_BINARY_SOURCE_OVERRIDE) VERSION="${SHORT_SHA}-previous-k0s"
           tar -C output/bin -czvf embedded-cluster-linux-amd64-previous-k0s.tgz embedded-cluster
@@ -74,7 +74,7 @@ jobs:
           REPLICATED_API_TOKEN: ${{ secrets.STAGING_REPLICATED_API_TOKEN }}
           REPLICATED_API_ORIGIN: "https://api.staging.replicated.com/vendor"
         run: |
-          export SHORT_SHA=dev-$(git rev-parse --short=7 HEAD^{commit})
+          export SHORT_SHA=dev-${GITHUB_SHA::7}
           echo "${SHORT_SHA}"
           sed -i "s/__version_string__/${SHORT_SHA}/g" e2e/kots-release-install/cluster-config.yaml
           sed -i "s/__version_string__/${SHORT_SHA}-upgrade/g" e2e/kots-release-upgrade/cluster-config.yaml
@@ -86,7 +86,7 @@ jobs:
           replicated release create --yaml-dir e2e/kots-release-upgrade --promote CI --version "appver-${SHORT_SHA}-upgrade"
       - name: Build CI binary
         run: |
-          export SHORT_SHA=dev-$(git rev-parse --short=7 HEAD^{commit})
+          export SHORT_SHA=dev-${GITHUB_SHA::7}
           export LOCAL_ARTIFACT_MIRROR_IMAGE=registry.staging.replicated.com/library/embedded-cluster-local-artifact-mirror
           echo "# channel release object" > e2e/kots-release-install/release.yaml
           echo 'channelID: "2cHXb1RCttzpR0xvnNWyaZCgDBP"' >> e2e/kots-release-install/release.yaml
@@ -105,7 +105,7 @@ jobs:
           AWS_SECRET_ACCESS_KEY: ${{ secrets.STAGING_EMBEDDED_CLUSTER_UPLOAD_IAM_SECRET }}
           AWS_REGION: "us-east-1"
         run: |
-          export SHORT_SHA=dev-$(git rev-parse --short=7 HEAD^{commit})
+          export SHORT_SHA=dev-${GITHUB_SHA::7}
           export EC_VERSION="v${SHORT_SHA}"
           ./scripts/cache-files.sh
           ./scripts/create-upgrade-release.sh
@@ -116,7 +116,7 @@ jobs:
           REPLICATED_API_TOKEN: ${{ secrets.STAGING_REPLICATED_API_TOKEN }}
           REPLICATED_API_ORIGIN: "https://api.staging.replicated.com/vendor"
         run: |
-          export SHORT_SHA=dev-$(git rev-parse --short=7 HEAD^{commit})
+          export SHORT_SHA=dev-${GITHUB_SHA::7}
           echo "${SHORT_SHA}"
           rm e2e/kots-release-install/release.yaml
           replicated release create --yaml-dir e2e/kots-release-install --promote CI-airgap --version "appver-${SHORT_SHA}"

From 1499c9e433bd8330d4f6a5d6baf1920a47a7166a Mon Sep 17 00:00:00 2001
From: Ethan Mosbaugh <ethan@replicated.com>
Date: Wed, 26 Jun 2024 20:17:36 -0700
Subject: [PATCH 17/96] correct commit sha

---
 .github/actions/e2e/action.yml      |  3 ++-
 .github/workflows/pull-request.yaml | 21 ++++++++++++++-------
 2 files changed, 16 insertions(+), 8 deletions(-)

diff --git a/.github/actions/e2e/action.yml b/.github/actions/e2e/action.yml
index 6edba565b..332e21920 100644
--- a/.github/actions/e2e/action.yml
+++ b/.github/actions/e2e/action.yml
@@ -89,7 +89,8 @@ runs:
   - name: E2E
     shell: bash
     run: |
-      export SHORT_SHA=dev-${GITHUB_SHA::7}
+      SHA=${{ github.event.pull_request.head.sha || github.sha }}
+      export SHORT_SHA=dev-${SHA::7}
       export LICENSE_ID=${{ inputs.license-id }}
       export AIRGAP_LICENSE_ID=${{ inputs.airgap-license-id }}
       export SNAPSHOT_LICENSE_ID=${{ inputs.snapshot-license-id }}
diff --git a/.github/workflows/pull-request.yaml b/.github/workflows/pull-request.yaml
index a8318b110..d389d7742 100644
--- a/.github/workflows/pull-request.yaml
+++ b/.github/workflows/pull-request.yaml
@@ -63,7 +63,8 @@ jobs:
 
     - name: Build and push local-artifact-mirror image
       run: |
-        export SHORT_SHA=dev-${GITHUB_SHA::7}
+        SHA=${{ github.event.pull_request.head.sha }}
+        export SHORT_SHA=dev-${SHA::7}
         export LOCAL_ARTIFACT_MIRROR_IMAGE=registry.staging.replicated.com/library/embedded-cluster-local-artifact-mirror
         make build-and-push-local-artifact-mirror-image VERSION="${SHORT_SHA}"
         make build-and-push-local-artifact-mirror-image VERSION="${SHORT_SHA}-previous-k0s"
@@ -71,7 +72,8 @@ jobs:
 
     - name: Build Linux AMD64 and Output Metadata
       run: |
-        export SHORT_SHA=dev-${GITHUB_SHA::7}
+        SHA=${{ github.event.pull_request.head.sha }}
+        export SHORT_SHA=dev-${SHA::7}
         export LOCAL_ARTIFACT_MIRROR_IMAGE=registry.staging.replicated.com/library/embedded-cluster-local-artifact-mirror
         make -B embedded-cluster-linux-amd64 K0S_VERSION=$(make print-PREVIOUS_K0S_VERSION) K0S_BINARY_SOURCE_OVERRIDE=$(make print-PREVIOUS_K0S_BINARY_SOURCE_OVERRIDE) VERSION="${SHORT_SHA}-previous-k0s"
         tar -C output/bin -czvf embedded-cluster-linux-amd64-previous-k0s.tgz embedded-cluster
@@ -98,7 +100,8 @@ jobs:
         REPLICATED_API_TOKEN: ${{ secrets.STAGING_REPLICATED_API_TOKEN }}
         REPLICATED_API_ORIGIN: "https://api.staging.replicated.com/vendor"
       run: |
-        export SHORT_SHA=dev-${GITHUB_SHA::7}
+        SHA=${{ github.event.pull_request.head.sha }}
+        export SHORT_SHA=dev-${SHA::7}
         echo "${SHORT_SHA}"
         sed -i "s/__version_string__/${SHORT_SHA}/g" e2e/kots-release-install/cluster-config.yaml
         sed -i "s/__version_string__/${SHORT_SHA}-upgrade/g" e2e/kots-release-upgrade/cluster-config.yaml
@@ -110,7 +113,8 @@ jobs:
         replicated release create --yaml-dir e2e/kots-release-upgrade --promote CI --version "appver-${SHORT_SHA}-upgrade"
     - name: Build CI binary
       run: |
-        export SHORT_SHA=dev-${GITHUB_SHA::7}
+        SHA=${{ github.event.pull_request.head.sha }}
+        export SHORT_SHA=dev-${SHA::7}
         export LOCAL_ARTIFACT_MIRROR_IMAGE=registry.staging.replicated.com/library/embedded-cluster-local-artifact-mirror
         echo "# channel release object" > e2e/kots-release-install/release.yaml
         echo 'channelID: "2cHXb1RCttzpR0xvnNWyaZCgDBP"' >> e2e/kots-release-install/release.yaml
@@ -129,7 +133,8 @@ jobs:
         AWS_SECRET_ACCESS_KEY: ${{ secrets.STAGING_EMBEDDED_CLUSTER_UPLOAD_IAM_SECRET }}
         AWS_REGION: "us-east-1"
       run: |
-        export SHORT_SHA=dev-${GITHUB_SHA::7}
+        SHA=${{ github.event.pull_request.head.sha }}
+        export SHORT_SHA=dev-${SHA::7}
         export EC_VERSION="v${SHORT_SHA}"
         ./scripts/cache-files.sh
         ./scripts/create-upgrade-release.sh
@@ -141,7 +146,8 @@ jobs:
         REPLICATED_API_TOKEN: ${{ secrets.STAGING_REPLICATED_API_TOKEN }}
         REPLICATED_API_ORIGIN: "https://api.staging.replicated.com/vendor"
       run: |
-        export SHORT_SHA=dev-${GITHUB_SHA::7}
+        SHA=${{ github.event.pull_request.head.sha }}
+        export SHORT_SHA=dev-${SHA::7}
         echo "${SHORT_SHA}"
         rm e2e/kots-release-install/release.yaml
         replicated release create --yaml-dir e2e/kots-release-install --promote CI-airgap --version "appver-${SHORT_SHA}"
@@ -154,7 +160,8 @@ jobs:
 
     - name: Create download link message text
       run: |
-        export SHORT_SHA=dev-${GITHUB_SHA::7}
+        SHA=${{ github.event.pull_request.head.sha }}
+        export SHORT_SHA=dev-${SHA::7}
         
         echo "This PR has been released (on staging) and is available for download with a embedded-cluster-smoke-test-staging-app [license ID](https://vendor.staging.replicated.com/apps/embedded-cluster-smoke-test-staging-app/customers?sort=name-asc)." > download-link.txt
         echo "" >> download-link.txt

From 46edb599879ab7091d8c50dc52622566a11d987d Mon Sep 17 00:00:00 2001
From: Ethan Mosbaugh <ethan@replicated.com>
Date: Wed, 26 Jun 2024 20:34:37 -0700
Subject: [PATCH 18/96] f

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index b0809a6e7..7564bb8d2 100644
--- a/Makefile
+++ b/Makefile
@@ -29,7 +29,7 @@ K0S_BINARY_SOURCE_OVERRIDE = https://ec-k0s-binaries.s3.amazonaws.com/k0s-v1.29.
 PREVIOUS_K0S_BINARY_SOURCE_OVERRIDE =
 TROUBLESHOOT_VERSION = v0.93.1
 KOTS_VERSION = alpha
-KOTS_BINARY_URL_OVERRIDE = https://salah-test.s3.us-east-1.amazonaws.com/kots.tar.gz?response-content-disposition=inline&X-Amz-Security-Token=IQoJb3JpZ2luX2VjEDEaCXVzLWVhc3QtMSJHMEUCIQDtUBgqGM6PoWnrOJCVUch%2B%2FYy1FfxStYEC8jOlwD8lxAIgHATMPlYkJOqWlKgEGOKkVUQ4NnJ1HKVne4sUnD9DCYMqpwMI2f%2F%2F%2F%2F%2F%2F%2F%2F%2F%2FARADGgw0MjkxMTQyMTQ1MjYiDMJ5ZBSl%2F52Wfba6Gyr7AlYYzef2IAgyLAoUb1wyJk1X9I0uhuYcotXungxdqxDGOcfFE4hV9d4i8WoZplmkryiGBzUw%2Bjzpku90%2FI4GLNh9nEx54s2LssMYZKhBIrVLtRP%2FL27V0j%2Fz2tkG5pvOan9Xtw76QNWNk0POPSqgrkXZqfralXAvIOL311fKAW75EU%2BqD1r4d87Rg3EgJWuDLSxi6Cfe8TPDqp756SdqDbDb4EDLZdxOu2sO1GsC5bcmEicQ1Nbwh4dt3pUosI1o2KGCJhTaMGexUcIEJEEd1ufMTXw%2BFgKelgDvqWInS9OKD8pfikwv9XjiXzJkDS0q4Xzt%2F1FEkU%2B19EpEdoNWebAdEsuo3rlKO1Yea4Q39ZQEz1r1fX8sH79VrZAwXqVidLw3RLo0VQPWT5Bc8HnDvL%2FEZguSNq8mku1wcWllhMQgu4hEK7LCQNO2Xa6KzKv%2FfTan3%2BS1qxQbK2MYL4TeGue9J9vC65V2hEpbIkgcwxqbnfspq2JDYgFgmOAwkYDxswY6hAIUo3WwdTORwSvxb8RPomy15E5O6BIuv%2BTD9nRuzjTTYVsWwOdQBj%2FlXuqHyfk3zxdXiW8DJhXAQdSsf1hKXMKgcF%2BgwnUnZO7mKve%2FteeRCenbJKDvdb0VSIiKaMiPoPUWt1uF1ISDbMDcS4PBYicigenAh0PJ8pOyOnR%2B9TZCqI%2BDMmfm9rKkOS1ZA9p0bx9daU3w1fMHurqjIQ0UA59qJTkJyDDnL1nfVoLOjCydDwgDB5puiKJK6Xnwy63n3IDHVhfkM40J02iAX%2FUvrDfu2f3bd8QGkK8xZjg5IpyIPDJ3yS%2BMmwO%2BDraxn40%2BZ21k5bngd0V6HZPDxpvlaCofeIY19A%3D%3D&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Date=20240626T162311Z&X-Amz-SignedHeaders=host&X-Amz-Expires=43200&X-Amz-Credential=ASIAWH2JTJB7JH7JTP6H%2F20240626%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Signature=6ec25e1182156abc30d97b20fd4843a0c6793d6795bbc0b032861d554bcb529b
+KOTS_BINARY_URL_OVERRIDE = https://repldev-ethan-test.s3.amazonaws.com/kots.tar.gz
 LOCAL_ARTIFACT_MIRROR_IMAGE ?= registry.replicated.com/library/embedded-cluster-local-artifact-mirror
 LOCAL_ARTIFACT_MIRROR_IMAGE_LOCATION = ${LOCAL_ARTIFACT_MIRROR_IMAGE}:$(subst +,-,$(VERSION))
 LD_FLAGS = -X github.com/replicatedhq/embedded-cluster/pkg/defaults.K0sVersion=$(K0S_VERSION) \

From acfb91a77c204add0baebebfcd96a4e0d85bc290 Mon Sep 17 00:00:00 2001
From: Ethan Mosbaugh <ethan@replicated.com>
Date: Wed, 26 Jun 2024 20:35:37 -0700
Subject: [PATCH 19/96] change version

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 7564bb8d2..9d646cc30 100644
--- a/Makefile
+++ b/Makefile
@@ -9,7 +9,7 @@ ADMIN_CONSOLE_MIGRATIONS_IMAGE_OVERRIDE =
 ADMIN_CONSOLE_KURL_PROXY_IMAGE_OVERRIDE = ttl.sh/ethan/kurl-proxy:24h
 EMBEDDED_OPERATOR_CHART_URL = oci://ttl.sh/ethan
 EMBEDDED_OPERATOR_CHART_NAME = embedded-cluster-operator
-EMBEDDED_OPERATOR_CHART_VERSION = 0.0.0-alpha.1
+EMBEDDED_OPERATOR_CHART_VERSION = 0.0.0-alpha.2
 EMBEDDED_OPERATOR_BINARY_URL_OVERRIDE = https://repldev-ethan-test.s3.amazonaws.com/manager
 EMBEDDED_OPERATOR_UTILS_IMAGE = busybox:1.36.1
 EMBEDDED_CLUSTER_OPERATOR_IMAGE_OVERRIDE =

From aaa21c3e85696f31dd4f2751876ef275db28c27a Mon Sep 17 00:00:00 2001
From: Ethan Mosbaugh <ethan@replicated.com>
Date: Thu, 27 Jun 2024 10:39:10 -0700
Subject: [PATCH 20/96] f

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 9d646cc30..05ccff4e1 100644
--- a/Makefile
+++ b/Makefile
@@ -9,7 +9,7 @@ ADMIN_CONSOLE_MIGRATIONS_IMAGE_OVERRIDE =
 ADMIN_CONSOLE_KURL_PROXY_IMAGE_OVERRIDE = ttl.sh/ethan/kurl-proxy:24h
 EMBEDDED_OPERATOR_CHART_URL = oci://ttl.sh/ethan
 EMBEDDED_OPERATOR_CHART_NAME = embedded-cluster-operator
-EMBEDDED_OPERATOR_CHART_VERSION = 0.0.0-alpha.2
+EMBEDDED_OPERATOR_CHART_VERSION = 0.0.0-alpha.3
 EMBEDDED_OPERATOR_BINARY_URL_OVERRIDE = https://repldev-ethan-test.s3.amazonaws.com/manager
 EMBEDDED_OPERATOR_UTILS_IMAGE = busybox:1.36.1
 EMBEDDED_CLUSTER_OPERATOR_IMAGE_OVERRIDE =

From 461132201f032d65b8bdf00eb181f20a5fef5753 Mon Sep 17 00:00:00 2001
From: Ethan Mosbaugh <ethan@replicated.com>
Date: Thu, 27 Jun 2024 11:12:49 -0700
Subject: [PATCH 21/96] f

---
 scripts/cache-files.sh | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/scripts/cache-files.sh b/scripts/cache-files.sh
index 3128ac44f..157efcf3f 100755
--- a/scripts/cache-files.sh
+++ b/scripts/cache-files.sh
@@ -70,6 +70,16 @@ function operatorbin() {
     local operator_version=
     operator_version=$(awk '/^EMBEDDED_OPERATOR_CHART_VERSION/{print $3}' Makefile)
 
+    # check if the binary already exists in the bucket
+    local operator_binary_exists=
+    operator_binary_exists=$(aws s3api head-object --bucket "${S3_BUCKET}" --key "operator-binaries/${operator_version}.tar.gz" || true)
+
+    # if the binary already exists, we don't need to upload it again
+    if [ -n "${operator_binary_exists}" ]; then
+        echo "operator binary ${operator_version} already exists in bucket ${S3_BUCKET}, skipping upload"
+        return 0
+    fi
+
     local operator_override=
     operator_override=$(awk '/^EMBEDDED_OPERATOR_BINARY_URL_OVERRIDE/{gsub("\"", "", $3); print $3}' Makefile)
 
@@ -81,16 +91,6 @@ function operatorbin() {
         echo "EMBEDDED_OPERATOR_BINARY_URL_OVERRIDE is set to '${operator_override}', using that source"
         curl --fail-with-body -L -o operator "${operator_override}"
     else
-        # check if the binary already exists in the bucket
-        local operator_binary_exists=
-        operator_binary_exists=$(aws s3api head-object --bucket "${S3_BUCKET}" --key "operator-binaries/${operator_version}.tar.gz" || true)
-
-        # if the binary already exists, we don't need to upload it again
-        if [ -n "${operator_binary_exists}" ]; then
-            echo "operator binary ${operator_version} already exists in bucket ${S3_BUCKET}, skipping upload"
-            return 0
-        fi
-
         # download the operator binary from github
         echo "downloading embedded cluster operator binary from https://github.com/replicatedhq/embedded-cluster-operator/releases/download/v${operator_version}/manager"
         curl --fail-with-body -L -o operator "https://github.com/replicatedhq/embedded-cluster-operator/releases/download/v${operator_version}/manager"

From 56055159bd5e918289be64990647d5307902f8e2 Mon Sep 17 00:00:00 2001
From: Ethan Mosbaugh <ethan@replicated.com>
Date: Thu, 27 Jun 2024 11:28:14 -0700
Subject: [PATCH 22/96] f

---
 Makefile               | 2 +-
 scripts/cache-files.sh | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 05ccff4e1..2f6debe4c 100644
--- a/Makefile
+++ b/Makefile
@@ -9,7 +9,7 @@ ADMIN_CONSOLE_MIGRATIONS_IMAGE_OVERRIDE =
 ADMIN_CONSOLE_KURL_PROXY_IMAGE_OVERRIDE = ttl.sh/ethan/kurl-proxy:24h
 EMBEDDED_OPERATOR_CHART_URL = oci://ttl.sh/ethan
 EMBEDDED_OPERATOR_CHART_NAME = embedded-cluster-operator
-EMBEDDED_OPERATOR_CHART_VERSION = 0.0.0-alpha.3
+EMBEDDED_OPERATOR_CHART_VERSION = 0.0.0-alpha.4
 EMBEDDED_OPERATOR_BINARY_URL_OVERRIDE = https://repldev-ethan-test.s3.amazonaws.com/manager
 EMBEDDED_OPERATOR_UTILS_IMAGE = busybox:1.36.1
 EMBEDDED_CLUSTER_OPERATOR_IMAGE_OVERRIDE =
diff --git a/scripts/cache-files.sh b/scripts/cache-files.sh
index 157efcf3f..76f90db39 100755
--- a/scripts/cache-files.sh
+++ b/scripts/cache-files.sh
@@ -75,6 +75,7 @@ function operatorbin() {
     operator_binary_exists=$(aws s3api head-object --bucket "${S3_BUCKET}" --key "operator-binaries/${operator_version}.tar.gz" || true)
 
     # if the binary already exists, we don't need to upload it again
+    # even if we did upload it again, cloudflare caches the file so its not guaranteed you will get the new file
     if [ -n "${operator_binary_exists}" ]; then
         echo "operator binary ${operator_version} already exists in bucket ${S3_BUCKET}, skipping upload"
         return 0

From e807fdd4eae9b45efb5406a8fe6357dad3ba71a3 Mon Sep 17 00:00:00 2001
From: Ethan Mosbaugh <ethan@replicated.com>
Date: Thu, 27 Jun 2024 12:50:31 -0700
Subject: [PATCH 23/96] f

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 2f6debe4c..58b0c5ddc 100644
--- a/Makefile
+++ b/Makefile
@@ -9,7 +9,7 @@ ADMIN_CONSOLE_MIGRATIONS_IMAGE_OVERRIDE =
 ADMIN_CONSOLE_KURL_PROXY_IMAGE_OVERRIDE = ttl.sh/ethan/kurl-proxy:24h
 EMBEDDED_OPERATOR_CHART_URL = oci://ttl.sh/ethan
 EMBEDDED_OPERATOR_CHART_NAME = embedded-cluster-operator
-EMBEDDED_OPERATOR_CHART_VERSION = 0.0.0-alpha.4
+EMBEDDED_OPERATOR_CHART_VERSION = 0.0.0-alpha.5
 EMBEDDED_OPERATOR_BINARY_URL_OVERRIDE = https://repldev-ethan-test.s3.amazonaws.com/manager
 EMBEDDED_OPERATOR_UTILS_IMAGE = busybox:1.36.1
 EMBEDDED_CLUSTER_OPERATOR_IMAGE_OVERRIDE =

From 07492b244875ff2290a3d7d333212333e71ef0fd Mon Sep 17 00:00:00 2001
From: Ethan Mosbaugh <ethan@replicated.com>
Date: Thu, 27 Jun 2024 13:36:13 -0700
Subject: [PATCH 24/96] d

---
 .github/workflows/pull-request.yaml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/pull-request.yaml b/.github/workflows/pull-request.yaml
index d389d7742..5e7166bcb 100644
--- a/.github/workflows/pull-request.yaml
+++ b/.github/workflows/pull-request.yaml
@@ -7,7 +7,7 @@ concurrency:
 jobs:
   sanitize:
     name: Sanitize
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-latest-4-cores
     steps:
     - name: Checkout
       uses: actions/checkout@v4
@@ -24,7 +24,7 @@ jobs:
         version: v1.54
   tests:
     name: Unit tests
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-latest-4-cores
     steps:
     - name: Checkout
       uses: actions/checkout@v4
@@ -40,7 +40,7 @@ jobs:
         make unit-tests
   build:
     name: Build
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-latest-4-cores
     permissions:
       pull-requests: write
     steps:

From db9baad36f46824e52d8f148a9fd7c02fe6e6ad4 Mon Sep 17 00:00:00 2001
From: Ethan Mosbaugh <ethan@replicated.com>
Date: Thu, 27 Jun 2024 13:36:36 -0700
Subject: [PATCH 25/96] d

---
 .github/workflows/pull-request.yaml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/pull-request.yaml b/.github/workflows/pull-request.yaml
index 5e7166bcb..d389d7742 100644
--- a/.github/workflows/pull-request.yaml
+++ b/.github/workflows/pull-request.yaml
@@ -7,7 +7,7 @@ concurrency:
 jobs:
   sanitize:
     name: Sanitize
-    runs-on: ubuntu-latest-4-cores
+    runs-on: ubuntu-latest
     steps:
     - name: Checkout
       uses: actions/checkout@v4
@@ -24,7 +24,7 @@ jobs:
         version: v1.54
   tests:
     name: Unit tests
-    runs-on: ubuntu-latest-4-cores
+    runs-on: ubuntu-latest
     steps:
     - name: Checkout
       uses: actions/checkout@v4
@@ -40,7 +40,7 @@ jobs:
         make unit-tests
   build:
     name: Build
-    runs-on: ubuntu-latest-4-cores
+    runs-on: ubuntu-latest
     permissions:
       pull-requests: write
     steps:

From bffa9e0197306c9ac8f2cdc5258f3a97f86e8fed Mon Sep 17 00:00:00 2001
From: Ethan Mosbaugh <ethan@replicated.com>
Date: Thu, 27 Jun 2024 13:58:20 -0700
Subject: [PATCH 26/96] f

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 58b0c5ddc..ffcb592a5 100644
--- a/Makefile
+++ b/Makefile
@@ -9,7 +9,7 @@ ADMIN_CONSOLE_MIGRATIONS_IMAGE_OVERRIDE =
 ADMIN_CONSOLE_KURL_PROXY_IMAGE_OVERRIDE = ttl.sh/ethan/kurl-proxy:24h
 EMBEDDED_OPERATOR_CHART_URL = oci://ttl.sh/ethan
 EMBEDDED_OPERATOR_CHART_NAME = embedded-cluster-operator
-EMBEDDED_OPERATOR_CHART_VERSION = 0.0.0-alpha.5
+EMBEDDED_OPERATOR_CHART_VERSION = 0.0.0-alpha.6
 EMBEDDED_OPERATOR_BINARY_URL_OVERRIDE = https://repldev-ethan-test.s3.amazonaws.com/manager
 EMBEDDED_OPERATOR_UTILS_IMAGE = busybox:1.36.1
 EMBEDDED_CLUSTER_OPERATOR_IMAGE_OVERRIDE =

From 63633cf92550968093179fd9740312d5a3c39667 Mon Sep 17 00:00:00 2001
From: Ethan Mosbaugh <ethan@replicated.com>
Date: Thu, 27 Jun 2024 14:41:19 -0700
Subject: [PATCH 27/96] f

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index ffcb592a5..15b1945b1 100644
--- a/Makefile
+++ b/Makefile
@@ -10,7 +10,7 @@ ADMIN_CONSOLE_KURL_PROXY_IMAGE_OVERRIDE = ttl.sh/ethan/kurl-proxy:24h
 EMBEDDED_OPERATOR_CHART_URL = oci://ttl.sh/ethan
 EMBEDDED_OPERATOR_CHART_NAME = embedded-cluster-operator
 EMBEDDED_OPERATOR_CHART_VERSION = 0.0.0-alpha.6
-EMBEDDED_OPERATOR_BINARY_URL_OVERRIDE = https://repldev-ethan-test.s3.amazonaws.com/manager
+EMBEDDED_OPERATOR_BINARY_URL_OVERRIDE = https://repldev-ethan-test.s3.amazonaws.com/manager-0.0.0-alpha.6
 EMBEDDED_OPERATOR_UTILS_IMAGE = busybox:1.36.1
 EMBEDDED_CLUSTER_OPERATOR_IMAGE_OVERRIDE =
 OPENEBS_CHART_VERSION = 4.1.0

From f36109d1b83151c5de030c0cddbb0e991c99d652 Mon Sep 17 00:00:00 2001
From: Ethan Mosbaugh <ethan@replicated.com>
Date: Thu, 27 Jun 2024 15:03:52 -0700
Subject: [PATCH 28/96] f

---
 Makefile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index 15b1945b1..5541db077 100644
--- a/Makefile
+++ b/Makefile
@@ -9,8 +9,8 @@ ADMIN_CONSOLE_MIGRATIONS_IMAGE_OVERRIDE =
 ADMIN_CONSOLE_KURL_PROXY_IMAGE_OVERRIDE = ttl.sh/ethan/kurl-proxy:24h
 EMBEDDED_OPERATOR_CHART_URL = oci://ttl.sh/ethan
 EMBEDDED_OPERATOR_CHART_NAME = embedded-cluster-operator
-EMBEDDED_OPERATOR_CHART_VERSION = 0.0.0-alpha.6
-EMBEDDED_OPERATOR_BINARY_URL_OVERRIDE = https://repldev-ethan-test.s3.amazonaws.com/manager-0.0.0-alpha.6
+EMBEDDED_OPERATOR_CHART_VERSION = 0.0.0-alpha.7
+EMBEDDED_OPERATOR_BINARY_URL_OVERRIDE = https://repldev-ethan-test.s3.amazonaws.com/manager-$(EMBEDDED_OPERATOR_CHART_VERSION)
 EMBEDDED_OPERATOR_UTILS_IMAGE = busybox:1.36.1
 EMBEDDED_CLUSTER_OPERATOR_IMAGE_OVERRIDE =
 OPENEBS_CHART_VERSION = 4.1.0

From cb9a34355d81fc26d79c58d04ffb8dcda968c1f4 Mon Sep 17 00:00:00 2001
From: Ethan Mosbaugh <ethan@replicated.com>
Date: Thu, 27 Jun 2024 15:10:50 -0700
Subject: [PATCH 29/96] f

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 5541db077..4ba06d2af 100644
--- a/Makefile
+++ b/Makefile
@@ -10,7 +10,7 @@ ADMIN_CONSOLE_KURL_PROXY_IMAGE_OVERRIDE = ttl.sh/ethan/kurl-proxy:24h
 EMBEDDED_OPERATOR_CHART_URL = oci://ttl.sh/ethan
 EMBEDDED_OPERATOR_CHART_NAME = embedded-cluster-operator
 EMBEDDED_OPERATOR_CHART_VERSION = 0.0.0-alpha.7
-EMBEDDED_OPERATOR_BINARY_URL_OVERRIDE = https://repldev-ethan-test.s3.amazonaws.com/manager-$(EMBEDDED_OPERATOR_CHART_VERSION)
+EMBEDDED_OPERATOR_BINARY_URL_OVERRIDE = https://repldev-ethan-test.s3.amazonaws.com/manager-0.0.0-alpha.7
 EMBEDDED_OPERATOR_UTILS_IMAGE = busybox:1.36.1
 EMBEDDED_CLUSTER_OPERATOR_IMAGE_OVERRIDE =
 OPENEBS_CHART_VERSION = 4.1.0

From fba9a8686146b63933a7a09495a51637ff05f779 Mon Sep 17 00:00:00 2001
From: Ethan Mosbaugh <ethan@replicated.com>
Date: Thu, 27 Jun 2024 15:28:36 -0700
Subject: [PATCH 30/96] f

---
 Makefile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index 4ba06d2af..dbfb8b32a 100644
--- a/Makefile
+++ b/Makefile
@@ -9,8 +9,8 @@ ADMIN_CONSOLE_MIGRATIONS_IMAGE_OVERRIDE =
 ADMIN_CONSOLE_KURL_PROXY_IMAGE_OVERRIDE = ttl.sh/ethan/kurl-proxy:24h
 EMBEDDED_OPERATOR_CHART_URL = oci://ttl.sh/ethan
 EMBEDDED_OPERATOR_CHART_NAME = embedded-cluster-operator
-EMBEDDED_OPERATOR_CHART_VERSION = 0.0.0-alpha.7
-EMBEDDED_OPERATOR_BINARY_URL_OVERRIDE = https://repldev-ethan-test.s3.amazonaws.com/manager-0.0.0-alpha.7
+EMBEDDED_OPERATOR_CHART_VERSION = 0.0.0-alpha.8
+EMBEDDED_OPERATOR_BINARY_URL_OVERRIDE = https://repldev-ethan-test.s3.amazonaws.com/manager-0.0.0-alpha.8
 EMBEDDED_OPERATOR_UTILS_IMAGE = busybox:1.36.1
 EMBEDDED_CLUSTER_OPERATOR_IMAGE_OVERRIDE =
 OPENEBS_CHART_VERSION = 4.1.0

From 0c77f38f1dbdce7b00f4b6a36db7cc948ba20038 Mon Sep 17 00:00:00 2001
From: Ethan Mosbaugh <ethan@replicated.com>
Date: Thu, 27 Jun 2024 22:31:00 -0700
Subject: [PATCH 31/96] f

---
 Makefile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index dbfb8b32a..d10bc4a33 100644
--- a/Makefile
+++ b/Makefile
@@ -9,8 +9,8 @@ ADMIN_CONSOLE_MIGRATIONS_IMAGE_OVERRIDE =
 ADMIN_CONSOLE_KURL_PROXY_IMAGE_OVERRIDE = ttl.sh/ethan/kurl-proxy:24h
 EMBEDDED_OPERATOR_CHART_URL = oci://ttl.sh/ethan
 EMBEDDED_OPERATOR_CHART_NAME = embedded-cluster-operator
-EMBEDDED_OPERATOR_CHART_VERSION = 0.0.0-alpha.8
-EMBEDDED_OPERATOR_BINARY_URL_OVERRIDE = https://repldev-ethan-test.s3.amazonaws.com/manager-0.0.0-alpha.8
+EMBEDDED_OPERATOR_CHART_VERSION = 0.0.0-alpha.9
+EMBEDDED_OPERATOR_BINARY_URL_OVERRIDE = https://repldev-ethan-test.s3.amazonaws.com/manager-0.0.0-alpha.9
 EMBEDDED_OPERATOR_UTILS_IMAGE = busybox:1.36.1
 EMBEDDED_CLUSTER_OPERATOR_IMAGE_OVERRIDE =
 OPENEBS_CHART_VERSION = 4.1.0

From db2c9974573744e362b6644b0678ef435ba12cd5 Mon Sep 17 00:00:00 2001
From: Ethan Mosbaugh <ethan@replicated.com>
Date: Fri, 28 Jun 2024 10:52:24 -0700
Subject: [PATCH 32/96] f

---
 Makefile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index d10bc4a33..add128d4e 100644
--- a/Makefile
+++ b/Makefile
@@ -9,8 +9,8 @@ ADMIN_CONSOLE_MIGRATIONS_IMAGE_OVERRIDE =
 ADMIN_CONSOLE_KURL_PROXY_IMAGE_OVERRIDE = ttl.sh/ethan/kurl-proxy:24h
 EMBEDDED_OPERATOR_CHART_URL = oci://ttl.sh/ethan
 EMBEDDED_OPERATOR_CHART_NAME = embedded-cluster-operator
-EMBEDDED_OPERATOR_CHART_VERSION = 0.0.0-alpha.9
-EMBEDDED_OPERATOR_BINARY_URL_OVERRIDE = https://repldev-ethan-test.s3.amazonaws.com/manager-0.0.0-alpha.9
+EMBEDDED_OPERATOR_CHART_VERSION = 0.0.0-alpha.10
+EMBEDDED_OPERATOR_BINARY_URL_OVERRIDE = https://repldev-ethan-test.s3.amazonaws.com/manager-0.0.0-alpha.10
 EMBEDDED_OPERATOR_UTILS_IMAGE = busybox:1.36.1
 EMBEDDED_CLUSTER_OPERATOR_IMAGE_OVERRIDE =
 OPENEBS_CHART_VERSION = 4.1.0

From 4a1d6a0311d5cc7bf32fc9034e852518eb19d747 Mon Sep 17 00:00:00 2001
From: Ethan Mosbaugh <ethan@replicated.com>
Date: Mon, 1 Jul 2024 12:34:43 -0700
Subject: [PATCH 33/96] f

---
 Makefile | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Makefile b/Makefile
index add128d4e..bc7cb2f93 100644
--- a/Makefile
+++ b/Makefile
@@ -7,10 +7,10 @@ ADMIN_CONSOLE_CHART_VERSION = 1.111.0
 ADMIN_CONSOLE_IMAGE_OVERRIDE = ttl.sh/ethan/kotsadm:24h
 ADMIN_CONSOLE_MIGRATIONS_IMAGE_OVERRIDE =
 ADMIN_CONSOLE_KURL_PROXY_IMAGE_OVERRIDE = ttl.sh/ethan/kurl-proxy:24h
-EMBEDDED_OPERATOR_CHART_URL = oci://ttl.sh/ethan
+EMBEDDED_OPERATOR_CHART_URL = oci://registry.replicated.com/library
 EMBEDDED_OPERATOR_CHART_NAME = embedded-cluster-operator
-EMBEDDED_OPERATOR_CHART_VERSION = 0.0.0-alpha.10
-EMBEDDED_OPERATOR_BINARY_URL_OVERRIDE = https://repldev-ethan-test.s3.amazonaws.com/manager-0.0.0-alpha.10
+EMBEDDED_OPERATOR_CHART_VERSION = v0.37.0-alpha.2
+EMBEDDED_OPERATOR_BINARY_URL_OVERRIDE =
 EMBEDDED_OPERATOR_UTILS_IMAGE = busybox:1.36.1
 EMBEDDED_CLUSTER_OPERATOR_IMAGE_OVERRIDE =
 OPENEBS_CHART_VERSION = 4.1.0

From a200f7c5fe8dd518e454dd2c85ad9a76d4ca6330 Mon Sep 17 00:00:00 2001
From: Ethan Mosbaugh <ethan@replicated.com>
Date: Mon, 1 Jul 2024 12:41:58 -0700
Subject: [PATCH 34/96] f

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index bc7cb2f93..89e29f6bd 100644
--- a/Makefile
+++ b/Makefile
@@ -9,7 +9,7 @@ ADMIN_CONSOLE_MIGRATIONS_IMAGE_OVERRIDE =
 ADMIN_CONSOLE_KURL_PROXY_IMAGE_OVERRIDE = ttl.sh/ethan/kurl-proxy:24h
 EMBEDDED_OPERATOR_CHART_URL = oci://registry.replicated.com/library
 EMBEDDED_OPERATOR_CHART_NAME = embedded-cluster-operator
-EMBEDDED_OPERATOR_CHART_VERSION = v0.37.0-alpha.2
+EMBEDDED_OPERATOR_CHART_VERSION = v0.37.0-alpha.3
 EMBEDDED_OPERATOR_BINARY_URL_OVERRIDE =
 EMBEDDED_OPERATOR_UTILS_IMAGE = busybox:1.36.1
 EMBEDDED_CLUSTER_OPERATOR_IMAGE_OVERRIDE =

From 9b91041e11d0ca7ecdeda0d48e0b4c1bf78f82dd Mon Sep 17 00:00:00 2001
From: Ethan Mosbaugh <ethan@replicated.com>
Date: Mon, 1 Jul 2024 12:57:36 -0700
Subject: [PATCH 35/96] f

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 89e29f6bd..52957fb84 100644
--- a/Makefile
+++ b/Makefile
@@ -9,7 +9,7 @@ ADMIN_CONSOLE_MIGRATIONS_IMAGE_OVERRIDE =
 ADMIN_CONSOLE_KURL_PROXY_IMAGE_OVERRIDE = ttl.sh/ethan/kurl-proxy:24h
 EMBEDDED_OPERATOR_CHART_URL = oci://registry.replicated.com/library
 EMBEDDED_OPERATOR_CHART_NAME = embedded-cluster-operator
-EMBEDDED_OPERATOR_CHART_VERSION = v0.37.0-alpha.3
+EMBEDDED_OPERATOR_CHART_VERSION = 0.37.0-alpha.3
 EMBEDDED_OPERATOR_BINARY_URL_OVERRIDE =
 EMBEDDED_OPERATOR_UTILS_IMAGE = busybox:1.36.1
 EMBEDDED_CLUSTER_OPERATOR_IMAGE_OVERRIDE =

From ef1593e949b4dafb6690bf778217ea3aaa4fb25c Mon Sep 17 00:00:00 2001
From: Ethan Mosbaugh <ethan@replicated.com>
Date: Mon, 1 Jul 2024 13:11:20 -0700
Subject: [PATCH 36/96] f

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 52957fb84..a0516c9e3 100644
--- a/Makefile
+++ b/Makefile
@@ -9,7 +9,7 @@ ADMIN_CONSOLE_MIGRATIONS_IMAGE_OVERRIDE =
 ADMIN_CONSOLE_KURL_PROXY_IMAGE_OVERRIDE = ttl.sh/ethan/kurl-proxy:24h
 EMBEDDED_OPERATOR_CHART_URL = oci://registry.replicated.com/library
 EMBEDDED_OPERATOR_CHART_NAME = embedded-cluster-operator
-EMBEDDED_OPERATOR_CHART_VERSION = 0.37.0-alpha.3
+EMBEDDED_OPERATOR_CHART_VERSION = 0.37.0-build.1
 EMBEDDED_OPERATOR_BINARY_URL_OVERRIDE =
 EMBEDDED_OPERATOR_UTILS_IMAGE = busybox:1.36.1
 EMBEDDED_CLUSTER_OPERATOR_IMAGE_OVERRIDE =

From d6b51c14f4785c2c0f7a5919d724722969e80f01 Mon Sep 17 00:00:00 2001
From: Ethan Mosbaugh <ethan@replicated.com>
Date: Wed, 3 Jul 2024 14:36:04 -0700
Subject: [PATCH 37/96] f

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index a0516c9e3..a2b56ff73 100644
--- a/Makefile
+++ b/Makefile
@@ -28,7 +28,7 @@ PREVIOUS_K0S_VERSION ?= v1.28.8+k0s.0
 K0S_BINARY_SOURCE_OVERRIDE = https://ec-k0s-binaries.s3.amazonaws.com/k0s-v1.29.5%2Bk0s.0-ec.0
 PREVIOUS_K0S_BINARY_SOURCE_OVERRIDE =
 TROUBLESHOOT_VERSION = v0.93.1
-KOTS_VERSION = alpha
+KOTS_VERSION = alpha.1
 KOTS_BINARY_URL_OVERRIDE = https://repldev-ethan-test.s3.amazonaws.com/kots.tar.gz
 LOCAL_ARTIFACT_MIRROR_IMAGE ?= registry.replicated.com/library/embedded-cluster-local-artifact-mirror
 LOCAL_ARTIFACT_MIRROR_IMAGE_LOCATION = ${LOCAL_ARTIFACT_MIRROR_IMAGE}:$(subst +,-,$(VERSION))

From a599016731f46e001b5e51a70058259f18f45b1b Mon Sep 17 00:00:00 2001
From: Ethan Mosbaugh <ethan@replicated.com>
Date: Wed, 3 Jul 2024 14:38:28 -0700
Subject: [PATCH 38/96] f

---
 scripts/cache-files.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/cache-files.sh b/scripts/cache-files.sh
index 76f90db39..b8f13b029 100755
--- a/scripts/cache-files.sh
+++ b/scripts/cache-files.sh
@@ -109,7 +109,7 @@ function operatorbin() {
 function kotsbin() {
     # first, figure out what version of kots is in the current build
     local kots_version=
-    kots_version=alpha
+    kots_version=alpha.1
     # kots_version=$(awk '/^ADMIN_CONSOLE_CHART_VERSION/{print $3}' Makefile)
     # kots_version=$(echo "${kots_version}" | sed 's/\([0-9]\+\.[0-9]\+\.[0-9]\+\).*/\1/')
     # kots_version=$(echo "v${kots_version}") #reinclude 'v' in kots version string

From 872b747b61038795f9a0877cfba0da9e4e28e2c4 Mon Sep 17 00:00:00 2001
From: Ethan Mosbaugh <ethan@replicated.com>
Date: Wed, 3 Jul 2024 14:43:12 -0700
Subject: [PATCH 39/96] f

---
 scripts/cache-files.sh | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/scripts/cache-files.sh b/scripts/cache-files.sh
index b8f13b029..109a6aaef 100755
--- a/scripts/cache-files.sh
+++ b/scripts/cache-files.sh
@@ -109,10 +109,7 @@ function operatorbin() {
 function kotsbin() {
     # first, figure out what version of kots is in the current build
     local kots_version=
-    kots_version=alpha.1
-    # kots_version=$(awk '/^ADMIN_CONSOLE_CHART_VERSION/{print $3}' Makefile)
-    # kots_version=$(echo "${kots_version}" | sed 's/\([0-9]\+\.[0-9]\+\.[0-9]\+\).*/\1/')
-    # kots_version=$(echo "v${kots_version}") #reinclude 'v' in kots version string
+    kots_version=$(awk '/^KOTS_VERSION/{print $3}' Makefile)
 
     local kots_override=
     kots_override=$(awk '/^KOTS_BINARY_URL_OVERRIDE/{gsub("\"", "", $3); print $3}' Makefile)

From 0ba3feda87c16b40cc5741d5b59fbba851a030e1 Mon Sep 17 00:00:00 2001
From: Salah Al Saleh <sg.alsaleh@gmail.com>
Date: Mon, 8 Jul 2024 19:18:16 +0000
Subject: [PATCH 40/96] kots ec1 nightly build

---
 Makefile | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/Makefile b/Makefile
index a2b56ff73..f258d658c 100644
--- a/Makefile
+++ b/Makefile
@@ -3,13 +3,13 @@ UNAME := $(shell uname)
 ARCH := $(shell uname -m)
 APP_NAME = embedded-cluster
 ADMIN_CONSOLE_CHART_REPO_OVERRIDE =
-ADMIN_CONSOLE_CHART_VERSION = 1.111.0
-ADMIN_CONSOLE_IMAGE_OVERRIDE = ttl.sh/ethan/kotsadm:24h
-ADMIN_CONSOLE_MIGRATIONS_IMAGE_OVERRIDE =
-ADMIN_CONSOLE_KURL_PROXY_IMAGE_OVERRIDE = ttl.sh/ethan/kurl-proxy:24h
+ADMIN_CONSOLE_CHART_VERSION = 1.110.0
+ADMIN_CONSOLE_IMAGE_OVERRIDE = kotsadm/kotsadm:v2024.7.8-ec1-nightly
+ADMIN_CONSOLE_MIGRATIONS_IMAGE_OVERRIDE = kotsadm/kotsadm-migrations:v2024.7.8-ec1-nightly
+ADMIN_CONSOLE_KURL_PROXY_IMAGE_OVERRIDE = kotsadm//kurl-proxy:v2024.7.8-ec1-nightly
 EMBEDDED_OPERATOR_CHART_URL = oci://registry.replicated.com/library
 EMBEDDED_OPERATOR_CHART_NAME = embedded-cluster-operator
-EMBEDDED_OPERATOR_CHART_VERSION = 0.37.0-build.1
+EMBEDDED_OPERATOR_CHART_VERSION = 0.39.0-build.1
 EMBEDDED_OPERATOR_BINARY_URL_OVERRIDE =
 EMBEDDED_OPERATOR_UTILS_IMAGE = busybox:1.36.1
 EMBEDDED_CLUSTER_OPERATOR_IMAGE_OVERRIDE =
@@ -28,8 +28,8 @@ PREVIOUS_K0S_VERSION ?= v1.28.8+k0s.0
 K0S_BINARY_SOURCE_OVERRIDE = https://ec-k0s-binaries.s3.amazonaws.com/k0s-v1.29.5%2Bk0s.0-ec.0
 PREVIOUS_K0S_BINARY_SOURCE_OVERRIDE =
 TROUBLESHOOT_VERSION = v0.93.1
-KOTS_VERSION = alpha.1
-KOTS_BINARY_URL_OVERRIDE = https://repldev-ethan-test.s3.amazonaws.com/kots.tar.gz
+KOTS_VERSION = v2024.7.8-ec1-nightly
+KOTS_BINARY_URL_OVERRIDE =
 LOCAL_ARTIFACT_MIRROR_IMAGE ?= registry.replicated.com/library/embedded-cluster-local-artifact-mirror
 LOCAL_ARTIFACT_MIRROR_IMAGE_LOCATION = ${LOCAL_ARTIFACT_MIRROR_IMAGE}:$(subst +,-,$(VERSION))
 LD_FLAGS = -X github.com/replicatedhq/embedded-cluster/pkg/defaults.K0sVersion=$(K0S_VERSION) \

From fee29221cc7f27dd603e4112abadd0ae2b565583 Mon Sep 17 00:00:00 2001
From: Salah Al Saleh <sg.alsaleh@gmail.com>
Date: Mon, 8 Jul 2024 19:41:10 +0000
Subject: [PATCH 41/96] kots ec2 nightly build

---
 Makefile | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/Makefile b/Makefile
index f258d658c..6b429a31c 100644
--- a/Makefile
+++ b/Makefile
@@ -4,11 +4,9 @@ ARCH := $(shell uname -m)
 APP_NAME = embedded-cluster
 ADMIN_CONSOLE_CHART_REPO_OVERRIDE =
 ADMIN_CONSOLE_CHART_VERSION = 1.110.0
-ADMIN_CONSOLE_IMAGE_OVERRIDE = kotsadm/kotsadm:v2024.7.8-ec1-nightly
-ADMIN_CONSOLE_MIGRATIONS_IMAGE_OVERRIDE = kotsadm/kotsadm-migrations:v2024.7.8-ec1-nightly
-ADMIN_CONSOLE_KURL_PROXY_IMAGE_OVERRIDE = kotsadm//kurl-proxy:v2024.7.8-ec1-nightly
-EMBEDDED_OPERATOR_CHART_URL = oci://registry.replicated.com/library
-EMBEDDED_OPERATOR_CHART_NAME = embedded-cluster-operator
+ADMIN_CONSOLE_IMAGE_OVERRIDE = kotsadm/kotsadm:v2024.7.8-ec2-nightly
+ADMIN_CONSOLE_MIGRATIONS_IMAGE_OVERRIDE = kotsadm/kotsadm-migrations:v2024.7.8-ec2-nightly
+ADMIN_CONSOLE_KURL_PROXY_IMAGE_OVERRIDE = kotsadm//kurl-proxy:v2024.7.8-ec2-nightly
 EMBEDDED_OPERATOR_CHART_VERSION = 0.39.0-build.1
 EMBEDDED_OPERATOR_BINARY_URL_OVERRIDE =
 EMBEDDED_OPERATOR_UTILS_IMAGE = busybox:1.36.1
@@ -28,7 +26,7 @@ PREVIOUS_K0S_VERSION ?= v1.28.8+k0s.0
 K0S_BINARY_SOURCE_OVERRIDE = https://ec-k0s-binaries.s3.amazonaws.com/k0s-v1.29.5%2Bk0s.0-ec.0
 PREVIOUS_K0S_BINARY_SOURCE_OVERRIDE =
 TROUBLESHOOT_VERSION = v0.93.1
-KOTS_VERSION = v2024.7.8-ec1-nightly
+KOTS_VERSION = v2024.7.8-ec2-nightly
 KOTS_BINARY_URL_OVERRIDE =
 LOCAL_ARTIFACT_MIRROR_IMAGE ?= registry.replicated.com/library/embedded-cluster-local-artifact-mirror
 LOCAL_ARTIFACT_MIRROR_IMAGE_LOCATION = ${LOCAL_ARTIFACT_MIRROR_IMAGE}:$(subst +,-,$(VERSION))

From 8407584bafab7980395ff6f70ec4fd14b7ca296b Mon Sep 17 00:00:00 2001
From: Salah Al Saleh <sg.alsaleh@gmail.com>
Date: Mon, 8 Jul 2024 19:48:23 +0000
Subject: [PATCH 42/96] kots ec1 nightly build - kurl proxy fix

---
 Makefile | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/Makefile b/Makefile
index 6b429a31c..d706497db 100644
--- a/Makefile
+++ b/Makefile
@@ -4,9 +4,9 @@ ARCH := $(shell uname -m)
 APP_NAME = embedded-cluster
 ADMIN_CONSOLE_CHART_REPO_OVERRIDE =
 ADMIN_CONSOLE_CHART_VERSION = 1.110.0
-ADMIN_CONSOLE_IMAGE_OVERRIDE = kotsadm/kotsadm:v2024.7.8-ec2-nightly
-ADMIN_CONSOLE_MIGRATIONS_IMAGE_OVERRIDE = kotsadm/kotsadm-migrations:v2024.7.8-ec2-nightly
-ADMIN_CONSOLE_KURL_PROXY_IMAGE_OVERRIDE = kotsadm//kurl-proxy:v2024.7.8-ec2-nightly
+ADMIN_CONSOLE_IMAGE_OVERRIDE = kotsadm/kotsadm:v2024.7.8-ec1-nightly
+ADMIN_CONSOLE_MIGRATIONS_IMAGE_OVERRIDE = kotsadm/kotsadm-migrations:v2024.7.8-ec1-nightly
+ADMIN_CONSOLE_KURL_PROXY_IMAGE_OVERRIDE = kotsadm/kurl-proxy:v2024.7.8-ec1-nightly
 EMBEDDED_OPERATOR_CHART_VERSION = 0.39.0-build.1
 EMBEDDED_OPERATOR_BINARY_URL_OVERRIDE =
 EMBEDDED_OPERATOR_UTILS_IMAGE = busybox:1.36.1
@@ -26,7 +26,7 @@ PREVIOUS_K0S_VERSION ?= v1.28.8+k0s.0
 K0S_BINARY_SOURCE_OVERRIDE = https://ec-k0s-binaries.s3.amazonaws.com/k0s-v1.29.5%2Bk0s.0-ec.0
 PREVIOUS_K0S_BINARY_SOURCE_OVERRIDE =
 TROUBLESHOOT_VERSION = v0.93.1
-KOTS_VERSION = v2024.7.8-ec2-nightly
+KOTS_VERSION = v2024.7.8-ec1-nightly
 KOTS_BINARY_URL_OVERRIDE =
 LOCAL_ARTIFACT_MIRROR_IMAGE ?= registry.replicated.com/library/embedded-cluster-local-artifact-mirror
 LOCAL_ARTIFACT_MIRROR_IMAGE_LOCATION = ${LOCAL_ARTIFACT_MIRROR_IMAGE}:$(subst +,-,$(VERSION))

From c9820a48785d5ad503fe6a1a342c7463930561aa Mon Sep 17 00:00:00 2001
From: Salah Al Saleh <sg.alsaleh@gmail.com>
Date: Mon, 8 Jul 2024 20:16:59 +0000
Subject: [PATCH 43/96] kots ec2 nightly build

---
 Makefile | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/Makefile b/Makefile
index d706497db..2303f9717 100644
--- a/Makefile
+++ b/Makefile
@@ -4,9 +4,9 @@ ARCH := $(shell uname -m)
 APP_NAME = embedded-cluster
 ADMIN_CONSOLE_CHART_REPO_OVERRIDE =
 ADMIN_CONSOLE_CHART_VERSION = 1.110.0
-ADMIN_CONSOLE_IMAGE_OVERRIDE = kotsadm/kotsadm:v2024.7.8-ec1-nightly
-ADMIN_CONSOLE_MIGRATIONS_IMAGE_OVERRIDE = kotsadm/kotsadm-migrations:v2024.7.8-ec1-nightly
-ADMIN_CONSOLE_KURL_PROXY_IMAGE_OVERRIDE = kotsadm/kurl-proxy:v2024.7.8-ec1-nightly
+ADMIN_CONSOLE_IMAGE_OVERRIDE = kotsadm/kotsadm:v2024.7.8-ec2-nightly
+ADMIN_CONSOLE_MIGRATIONS_IMAGE_OVERRIDE = kotsadm/kotsadm-migrations:v2024.7.8-ec2-nightly
+ADMIN_CONSOLE_KURL_PROXY_IMAGE_OVERRIDE = kotsadm/kurl-proxy:v2024.7.8-ec2-nightly
 EMBEDDED_OPERATOR_CHART_VERSION = 0.39.0-build.1
 EMBEDDED_OPERATOR_BINARY_URL_OVERRIDE =
 EMBEDDED_OPERATOR_UTILS_IMAGE = busybox:1.36.1
@@ -26,7 +26,7 @@ PREVIOUS_K0S_VERSION ?= v1.28.8+k0s.0
 K0S_BINARY_SOURCE_OVERRIDE = https://ec-k0s-binaries.s3.amazonaws.com/k0s-v1.29.5%2Bk0s.0-ec.0
 PREVIOUS_K0S_BINARY_SOURCE_OVERRIDE =
 TROUBLESHOOT_VERSION = v0.93.1
-KOTS_VERSION = v2024.7.8-ec1-nightly
+KOTS_VERSION = v2024.7.8-ec2-nightly
 KOTS_BINARY_URL_OVERRIDE =
 LOCAL_ARTIFACT_MIRROR_IMAGE ?= registry.replicated.com/library/embedded-cluster-local-artifact-mirror
 LOCAL_ARTIFACT_MIRROR_IMAGE_LOCATION = ${LOCAL_ARTIFACT_MIRROR_IMAGE}:$(subst +,-,$(VERSION))

From da3bd2d312aca55385c30776cec0066a9b07a460 Mon Sep 17 00:00:00 2001
From: Salah Al Saleh <sg.alsaleh@gmail.com>
Date: Mon, 8 Jul 2024 22:51:36 +0000
Subject: [PATCH 44/96] kots v2024.7.8-ec5-nightly

---
 Makefile | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/Makefile b/Makefile
index 2303f9717..73ab3b262 100644
--- a/Makefile
+++ b/Makefile
@@ -4,9 +4,9 @@ ARCH := $(shell uname -m)
 APP_NAME = embedded-cluster
 ADMIN_CONSOLE_CHART_REPO_OVERRIDE =
 ADMIN_CONSOLE_CHART_VERSION = 1.110.0
-ADMIN_CONSOLE_IMAGE_OVERRIDE = kotsadm/kotsadm:v2024.7.8-ec2-nightly
-ADMIN_CONSOLE_MIGRATIONS_IMAGE_OVERRIDE = kotsadm/kotsadm-migrations:v2024.7.8-ec2-nightly
-ADMIN_CONSOLE_KURL_PROXY_IMAGE_OVERRIDE = kotsadm/kurl-proxy:v2024.7.8-ec2-nightly
+ADMIN_CONSOLE_IMAGE_OVERRIDE = kotsadm/kotsadm:v2024.7.8-ec5-nightly
+ADMIN_CONSOLE_MIGRATIONS_IMAGE_OVERRIDE = kotsadm/kotsadm-migrations:v2024.7.8-ec5-nightly
+ADMIN_CONSOLE_KURL_PROXY_IMAGE_OVERRIDE = kotsadm/kurl-proxy:v2024.7.8-ec5-nightly
 EMBEDDED_OPERATOR_CHART_VERSION = 0.39.0-build.1
 EMBEDDED_OPERATOR_BINARY_URL_OVERRIDE =
 EMBEDDED_OPERATOR_UTILS_IMAGE = busybox:1.36.1
@@ -26,7 +26,7 @@ PREVIOUS_K0S_VERSION ?= v1.28.8+k0s.0
 K0S_BINARY_SOURCE_OVERRIDE = https://ec-k0s-binaries.s3.amazonaws.com/k0s-v1.29.5%2Bk0s.0-ec.0
 PREVIOUS_K0S_BINARY_SOURCE_OVERRIDE =
 TROUBLESHOOT_VERSION = v0.93.1
-KOTS_VERSION = v2024.7.8-ec2-nightly
+KOTS_VERSION = v2024.7.8-ec5-nightly
 KOTS_BINARY_URL_OVERRIDE =
 LOCAL_ARTIFACT_MIRROR_IMAGE ?= registry.replicated.com/library/embedded-cluster-local-artifact-mirror
 LOCAL_ARTIFACT_MIRROR_IMAGE_LOCATION = ${LOCAL_ARTIFACT_MIRROR_IMAGE}:$(subst +,-,$(VERSION))

From db6359d24b6ab848dc62e7c2fa59dd5c205f771d Mon Sep 17 00:00:00 2001
From: Salah Al Saleh <sg.alsaleh@gmail.com>
Date: Mon, 8 Jul 2024 23:07:39 +0000
Subject: [PATCH 45/96] kots v2024.7.8-ec6-nightly

---
 Makefile | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/Makefile b/Makefile
index 73ab3b262..4a5c18ea2 100644
--- a/Makefile
+++ b/Makefile
@@ -4,9 +4,9 @@ ARCH := $(shell uname -m)
 APP_NAME = embedded-cluster
 ADMIN_CONSOLE_CHART_REPO_OVERRIDE =
 ADMIN_CONSOLE_CHART_VERSION = 1.110.0
-ADMIN_CONSOLE_IMAGE_OVERRIDE = kotsadm/kotsadm:v2024.7.8-ec5-nightly
-ADMIN_CONSOLE_MIGRATIONS_IMAGE_OVERRIDE = kotsadm/kotsadm-migrations:v2024.7.8-ec5-nightly
-ADMIN_CONSOLE_KURL_PROXY_IMAGE_OVERRIDE = kotsadm/kurl-proxy:v2024.7.8-ec5-nightly
+ADMIN_CONSOLE_IMAGE_OVERRIDE = kotsadm/kotsadm:v2024.7.8-ec6-nightly
+ADMIN_CONSOLE_MIGRATIONS_IMAGE_OVERRIDE = kotsadm/kotsadm-migrations:v2024.7.8-ec6-nightly
+ADMIN_CONSOLE_KURL_PROXY_IMAGE_OVERRIDE = kotsadm/kurl-proxy:v2024.7.8-ec6-nightly
 EMBEDDED_OPERATOR_CHART_VERSION = 0.39.0-build.1
 EMBEDDED_OPERATOR_BINARY_URL_OVERRIDE =
 EMBEDDED_OPERATOR_UTILS_IMAGE = busybox:1.36.1
@@ -26,7 +26,7 @@ PREVIOUS_K0S_VERSION ?= v1.28.8+k0s.0
 K0S_BINARY_SOURCE_OVERRIDE = https://ec-k0s-binaries.s3.amazonaws.com/k0s-v1.29.5%2Bk0s.0-ec.0
 PREVIOUS_K0S_BINARY_SOURCE_OVERRIDE =
 TROUBLESHOOT_VERSION = v0.93.1
-KOTS_VERSION = v2024.7.8-ec5-nightly
+KOTS_VERSION = v2024.7.8-ec6-nightly
 KOTS_BINARY_URL_OVERRIDE =
 LOCAL_ARTIFACT_MIRROR_IMAGE ?= registry.replicated.com/library/embedded-cluster-local-artifact-mirror
 LOCAL_ARTIFACT_MIRROR_IMAGE_LOCATION = ${LOCAL_ARTIFACT_MIRROR_IMAGE}:$(subst +,-,$(VERSION))

From a209094c79b6fd911a85aa0cdb32dd3fedb2af68 Mon Sep 17 00:00:00 2001
From: Salah Al Saleh <sg.alsaleh@gmail.com>
Date: Mon, 8 Jul 2024 23:37:49 +0000
Subject: [PATCH 46/96] kots v2024.7.8-ec7-nightly

---
 Makefile | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/Makefile b/Makefile
index 4a5c18ea2..802473f6a 100644
--- a/Makefile
+++ b/Makefile
@@ -4,10 +4,10 @@ ARCH := $(shell uname -m)
 APP_NAME = embedded-cluster
 ADMIN_CONSOLE_CHART_REPO_OVERRIDE =
 ADMIN_CONSOLE_CHART_VERSION = 1.110.0
-ADMIN_CONSOLE_IMAGE_OVERRIDE = kotsadm/kotsadm:v2024.7.8-ec6-nightly
-ADMIN_CONSOLE_MIGRATIONS_IMAGE_OVERRIDE = kotsadm/kotsadm-migrations:v2024.7.8-ec6-nightly
-ADMIN_CONSOLE_KURL_PROXY_IMAGE_OVERRIDE = kotsadm/kurl-proxy:v2024.7.8-ec6-nightly
-EMBEDDED_OPERATOR_CHART_VERSION = 0.39.0-build.1
+ADMIN_CONSOLE_IMAGE_OVERRIDE = kotsadm/kotsadm:v2024.7.8-ec7-nightly
+ADMIN_CONSOLE_MIGRATIONS_IMAGE_OVERRIDE = kotsadm/kotsadm-migrations:v2024.7.8-ec7-nightly
+ADMIN_CONSOLE_KURL_PROXY_IMAGE_OVERRIDE = kotsadm/kurl-proxy:v2024.7.8-ec7-nightly
+EMBEDDED_OPERATOR_CHART_VERSION = 0.37.0-build.1
 EMBEDDED_OPERATOR_BINARY_URL_OVERRIDE =
 EMBEDDED_OPERATOR_UTILS_IMAGE = busybox:1.36.1
 EMBEDDED_CLUSTER_OPERATOR_IMAGE_OVERRIDE =
@@ -26,7 +26,7 @@ PREVIOUS_K0S_VERSION ?= v1.28.8+k0s.0
 K0S_BINARY_SOURCE_OVERRIDE = https://ec-k0s-binaries.s3.amazonaws.com/k0s-v1.29.5%2Bk0s.0-ec.0
 PREVIOUS_K0S_BINARY_SOURCE_OVERRIDE =
 TROUBLESHOOT_VERSION = v0.93.1
-KOTS_VERSION = v2024.7.8-ec6-nightly
+KOTS_VERSION = v2024.7.8-ec7-nightly
 KOTS_BINARY_URL_OVERRIDE =
 LOCAL_ARTIFACT_MIRROR_IMAGE ?= registry.replicated.com/library/embedded-cluster-local-artifact-mirror
 LOCAL_ARTIFACT_MIRROR_IMAGE_LOCATION = ${LOCAL_ARTIFACT_MIRROR_IMAGE}:$(subst +,-,$(VERSION))

From b7d2148ef3704064cbcf59323f0a390c3bfb3981 Mon Sep 17 00:00:00 2001
From: Salah Al Saleh <sg.alsaleh@gmail.com>
Date: Tue, 9 Jul 2024 00:04:19 +0000
Subject: [PATCH 47/96] kots v2024.7.8-ec8-nightly

---
 Makefile | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/Makefile b/Makefile
index 802473f6a..cdcc64574 100644
--- a/Makefile
+++ b/Makefile
@@ -4,9 +4,9 @@ ARCH := $(shell uname -m)
 APP_NAME = embedded-cluster
 ADMIN_CONSOLE_CHART_REPO_OVERRIDE =
 ADMIN_CONSOLE_CHART_VERSION = 1.110.0
-ADMIN_CONSOLE_IMAGE_OVERRIDE = kotsadm/kotsadm:v2024.7.8-ec7-nightly
-ADMIN_CONSOLE_MIGRATIONS_IMAGE_OVERRIDE = kotsadm/kotsadm-migrations:v2024.7.8-ec7-nightly
-ADMIN_CONSOLE_KURL_PROXY_IMAGE_OVERRIDE = kotsadm/kurl-proxy:v2024.7.8-ec7-nightly
+ADMIN_CONSOLE_IMAGE_OVERRIDE = kotsadm/kotsadm:v2024.7.8-ec8-nightly
+ADMIN_CONSOLE_MIGRATIONS_IMAGE_OVERRIDE = kotsadm/kotsadm-migrations:v2024.7.8-ec8-nightly
+ADMIN_CONSOLE_KURL_PROXY_IMAGE_OVERRIDE = kotsadm/kurl-proxy:v2024.7.8-ec8-nightly
 EMBEDDED_OPERATOR_CHART_VERSION = 0.37.0-build.1
 EMBEDDED_OPERATOR_BINARY_URL_OVERRIDE =
 EMBEDDED_OPERATOR_UTILS_IMAGE = busybox:1.36.1
@@ -26,7 +26,7 @@ PREVIOUS_K0S_VERSION ?= v1.28.8+k0s.0
 K0S_BINARY_SOURCE_OVERRIDE = https://ec-k0s-binaries.s3.amazonaws.com/k0s-v1.29.5%2Bk0s.0-ec.0
 PREVIOUS_K0S_BINARY_SOURCE_OVERRIDE =
 TROUBLESHOOT_VERSION = v0.93.1
-KOTS_VERSION = v2024.7.8-ec7-nightly
+KOTS_VERSION = v2024.7.8-ec8-nightly
 KOTS_BINARY_URL_OVERRIDE =
 LOCAL_ARTIFACT_MIRROR_IMAGE ?= registry.replicated.com/library/embedded-cluster-local-artifact-mirror
 LOCAL_ARTIFACT_MIRROR_IMAGE_LOCATION = ${LOCAL_ARTIFACT_MIRROR_IMAGE}:$(subst +,-,$(VERSION))

From 7fb2a46e76a1ec1e3c1fbdcb0051e54ba4b18260 Mon Sep 17 00:00:00 2001
From: Salah Al Saleh <sg.alsaleh@gmail.com>
Date: Tue, 9 Jul 2024 00:30:13 +0000
Subject: [PATCH 48/96] operator 0.39.0-build.2

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index cdcc64574..3e49f250c 100644
--- a/Makefile
+++ b/Makefile
@@ -7,7 +7,7 @@ ADMIN_CONSOLE_CHART_VERSION = 1.110.0
 ADMIN_CONSOLE_IMAGE_OVERRIDE = kotsadm/kotsadm:v2024.7.8-ec8-nightly
 ADMIN_CONSOLE_MIGRATIONS_IMAGE_OVERRIDE = kotsadm/kotsadm-migrations:v2024.7.8-ec8-nightly
 ADMIN_CONSOLE_KURL_PROXY_IMAGE_OVERRIDE = kotsadm/kurl-proxy:v2024.7.8-ec8-nightly
-EMBEDDED_OPERATOR_CHART_VERSION = 0.37.0-build.1
+EMBEDDED_OPERATOR_CHART_VERSION = 0.39.0-build.2
 EMBEDDED_OPERATOR_BINARY_URL_OVERRIDE =
 EMBEDDED_OPERATOR_UTILS_IMAGE = busybox:1.36.1
 EMBEDDED_CLUSTER_OPERATOR_IMAGE_OVERRIDE =

From 52490d3f3c41193c3f5bf26311e5bcdf827fab16 Mon Sep 17 00:00:00 2001
From: Ethan Mosbaugh <ethan@replicated.com>
Date: Tue, 9 Jul 2024 14:12:22 -0700
Subject: [PATCH 49/96] upgrade test

---
 .github/actions/e2e/action.yml                |  1 +
 .github/workflows/pull-request.yaml           | 89 ++++++++++---------
 .github/workflows/release-dev.yaml            |  1 +
 e2e/install_test.go                           | 53 +++++++++++
 .../tests/deploy-upgrade/test.spec.ts         | 21 +++++
 e2e/playwright/tsconfig.json                  |  7 ++
 e2e/scripts/playwright.sh                     |  2 +
 7 files changed, 130 insertions(+), 44 deletions(-)
 create mode 100644 e2e/playwright/tests/deploy-upgrade/test.spec.ts
 create mode 100644 e2e/playwright/tsconfig.json

diff --git a/.github/actions/e2e/action.yml b/.github/actions/e2e/action.yml
index 332e21920..5fb08fef0 100644
--- a/.github/actions/e2e/action.yml
+++ b/.github/actions/e2e/action.yml
@@ -104,6 +104,7 @@ runs:
       export DR_AWS_S3_PREFIX_AIRGAP=${{ inputs.test-name }}-${{ github.run_id }}-${{ github.run_attempt }}-airgap
       export DR_AWS_ACCESS_KEY_ID=${{ inputs.dr-aws-access-key-id }}
       export DR_AWS_SECRET_ACCESS_KEY=${{ inputs.dr-aws-secret-access-key }}
+      export APP_UPGRADE_VERSION=appver-dev-${SHORT_SHA}-upgrade
       make e2e-test TEST_NAME=${{ inputs.test-name }}
   - name: Upload Host Support Bundle
     uses: actions/upload-artifact@v4
diff --git a/.github/workflows/pull-request.yaml b/.github/workflows/pull-request.yaml
index d389d7742..d30eabd8d 100644
--- a/.github/workflows/pull-request.yaml
+++ b/.github/workflows/pull-request.yaml
@@ -199,50 +199,51 @@ jobs:
       fail-fast: false
       matrix:
         test:
-          - TestSingleNodeInstallation
-          - TestSingleNodeInstallationAlmaLinux8
-          - TestSingleNodeInstallationDebian11
-          - TestSingleNodeInstallationDebian12
-          - TestSingleNodeInstallationCentos9Stream
-          - TestVersion
-          - TestHostPreflight
-          - TestUnsupportedOverrides
-          - TestMultiNodeInstallation
-          - TestMultiNodeReset
-          - TestCommandsRequireSudo
-          - TestInstallWithoutEmbed
-          - TestInstallFromReplicatedApp
-          - TestResetAndReinstall
-          - TestResetAndReinstallAirgap
-          - TestCollectSupportBundle
-          - TestOldVersionUpgrade
-          - TestMaterialize
-          - TestLocalArtifactMirror
-          - TestSingleNodeAirgapUpgrade
-          - TestSingleNodeAirgapUpgradeCustomCIDR
-          - TestInstallSnapshotFromReplicatedApp
-          - TestMultiNodeAirgapUpgrade
-          - TestSingleNodeDisasterRecovery
-          - TestSingleNodeDisasterRecoveryWithProxy
-          - TestSingleNodeResumeDisasterRecovery
-          - TestSingleNodeAirgapDisasterRecovery
-          - TestMultiNodeHAInstallation
-          - TestMultiNodeAirgapHAInstallation
-          - TestMultiNodeAirgapUpgradeSameK0s
-          - TestProxiedEnvironment
-          - TestMultiNodeHADisasterRecovery
-          - TestMultiNodeAirgapHADisasterRecovery
-          - TestCustomCIDR
-          - TestProxiedCustomCIDR
-        include:
-          - test: TestMultiNodeAirgapUpgrade
-            runner: embedded-cluster
-          - test: TestMultiNodeAirgapHAInstallation
-            runner: embedded-cluster
-          - test: TestMultiNodeAirgapHADisasterRecovery
-            runner: embedded-cluster
-          - test: TestSingleNodeAirgapDisasterRecovery
-            runner: embedded-cluster
+          # - TestSingleNodeInstallation
+          # - TestSingleNodeInstallationAlmaLinux8
+          # - TestSingleNodeInstallationDebian11
+          # - TestSingleNodeInstallationDebian12
+          # - TestSingleNodeInstallationCentos9Stream
+          - TestSingleNodeUpgrade
+        #   - TestVersion
+        #   - TestHostPreflight
+        #   - TestUnsupportedOverrides
+        #   - TestMultiNodeInstallation
+        #   - TestMultiNodeReset
+        #   - TestCommandsRequireSudo
+        #   - TestInstallWithoutEmbed
+        #   - TestInstallFromReplicatedApp
+        #   - TestResetAndReinstall
+        #   - TestResetAndReinstallAirgap
+        #   - TestCollectSupportBundle
+        #   - TestOldVersionUpgrade
+        #   - TestMaterialize
+        #   - TestLocalArtifactMirror
+        #   - TestSingleNodeAirgapUpgrade
+        #   - TestSingleNodeAirgapUpgradeCustomCIDR
+        #   - TestInstallSnapshotFromReplicatedApp
+        #   - TestMultiNodeAirgapUpgrade
+        #   - TestSingleNodeDisasterRecovery
+        #   - TestSingleNodeDisasterRecoveryWithProxy
+        #   - TestSingleNodeResumeDisasterRecovery
+        #   - TestSingleNodeAirgapDisasterRecovery
+        #   - TestMultiNodeHAInstallation
+        #   - TestMultiNodeAirgapHAInstallation
+        #   - TestMultiNodeAirgapUpgradeSameK0s
+        #   - TestProxiedEnvironment
+        #   - TestMultiNodeHADisasterRecovery
+        #   - TestMultiNodeAirgapHADisasterRecovery
+        #   - TestCustomCIDR
+        #   - TestProxiedCustomCIDR
+        include: []
+        #   - test: TestMultiNodeAirgapUpgrade
+        #     runner: embedded-cluster
+        #   - test: TestMultiNodeAirgapHAInstallation
+        #     runner: embedded-cluster
+        #   - test: TestMultiNodeAirgapHADisasterRecovery
+        #     runner: embedded-cluster
+        #   - test: TestSingleNodeAirgapDisasterRecovery
+        #     runner: embedded-cluster
     steps:
     - name: Checkout
       uses: actions/checkout@v4
diff --git a/.github/workflows/release-dev.yaml b/.github/workflows/release-dev.yaml
index bfd939c9d..080838758 100644
--- a/.github/workflows/release-dev.yaml
+++ b/.github/workflows/release-dev.yaml
@@ -149,6 +149,7 @@ jobs:
           - TestSingleNodeInstallationDebian11
           - TestSingleNodeInstallationDebian12
           - TestSingleNodeInstallationCentos9Stream
+          - TestSingleNodeUpgrade
           - TestVersion
           - TestHostPreflight
           - TestUnsupportedOverrides
diff --git a/e2e/install_test.go b/e2e/install_test.go
index ecd889604..380a10754 100644
--- a/e2e/install_test.go
+++ b/e2e/install_test.go
@@ -573,6 +573,59 @@ func TestOldVersionUpgrade(t *testing.T) {
 	t.Logf("%s: test complete", time.Now().Format(time.RFC3339))
 }
 
+func TestSingleNodeUpgrade(t *testing.T) {
+	t.Parallel()
+
+	requiredEnvVars := []string{
+		"APP_UPGRADE_VERSION",
+	}
+	for _, envVar := range requiredEnvVars {
+		if os.Getenv(envVar) == "" {
+			t.Fatalf("missing required environment variable: %s", envVar)
+		}
+	}
+
+	tc := cluster.NewTestCluster(&cluster.Input{
+		T:                   t,
+		Nodes:               1,
+		Image:               "ubuntu/jammy",
+		LicensePath:         "license.yaml",
+		EmbeddedClusterPath: "../output/bin/embedded-cluster",
+	})
+	defer cleanupCluster(t, tc)
+
+	t.Logf("%s: installing embedded-cluster on node 0", time.Now().Format(time.RFC3339))
+	line := []string{"single-node-install.sh", "ui"}
+	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
+		t.Fatalf("fail to install embedded-cluster on node %s: %v", tc.Nodes[0], err)
+	}
+
+	if err := setupPlaywright(t, tc); err != nil {
+		t.Fatalf("fail to setup playwright: %v", err)
+	}
+	if _, _, err := runPlaywrightTest(t, tc, "deploy-app"); err != nil {
+		t.Fatalf("fail to run playwright test deploy-app: %v", err)
+	}
+
+	t.Logf("%s: checking installation state", time.Now().Format(time.RFC3339))
+	line = []string{"check-installation-state.sh", os.Getenv("SHORT_SHA")}
+	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
+		t.Fatalf("fail to check installation state: %v", err)
+	}
+
+	testArgs := []string{}
+	for _, envVar := range requiredEnvVars {
+		testArgs = append(testArgs, os.Getenv(envVar))
+	}
+
+	t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339))
+	if _, _, err := runPlaywrightTest(t, tc, "deploy-upgrade", testArgs...); err != nil {
+		t.Fatalf("fail to run playwright test deploy-app: %v", err)
+	}
+
+	t.Logf("%s: test complete", time.Now().Format(time.RFC3339))
+}
+
 func TestSingleNodeAirgapUpgrade(t *testing.T) {
 	t.Parallel()
 
diff --git a/e2e/playwright/tests/deploy-upgrade/test.spec.ts b/e2e/playwright/tests/deploy-upgrade/test.spec.ts
new file mode 100644
index 000000000..e30a0ecef
--- /dev/null
+++ b/e2e/playwright/tests/deploy-upgrade/test.spec.ts
@@ -0,0 +1,21 @@
+import { test, expect } from '@playwright/test';
+import { login } from '../shared';
+
+test('deploy upgrade', async ({ page }) => {
+  test.setTimeout(15 * 60 * 1000); // 15 minutes
+  await login(page);
+  await page.getByRole('link', { name: 'Version history', exact: true }).click();
+  const rowLocator = '../../..'; // '//./ancestor::div[@class="available-update-row"]'
+  await page.getByText(process.env.APP_UPGRADE_VERSION, { exact: true })
+    .locator(rowLocator)
+    .getByRole('button', { name: 'Deploy', exact: true }).click();
+  const iframe = page.frameLocator('#upgrade-service-iframe');
+  await expect(iframe.locator('.ConfigArea--wrapper')).toBeVisible({ timeout: 20 * 1000 });
+  await iframe.getByRole('button', { name: 'Next', exact: true }).click();
+  await iframe.getByRole('button', { name: 'Next: Confirm and deploy', exact: true }).click({ timeout: 10 * 1000 });
+  await iframe.getByRole('button', { name: 'Deploy', exact: true }).click();
+  await expect(page.locator('.Modal-body').getByText('Cluster update in progress')).toBeVisible();
+  await expect(page.locator('.Modal-body').getByText('Cluster update in progress')).not.toBeVisible({ timeout: 60 * 1000 });
+  await page.getByRole('link', { name: 'Dashboard', exact: true }).click();
+  await expect(page.locator('.VersionCard-content--wrapper')).toContainText(process.env.APP_UPGRADE_VERSION);
+});
diff --git a/e2e/playwright/tsconfig.json b/e2e/playwright/tsconfig.json
new file mode 100644
index 000000000..1d6068c00
--- /dev/null
+++ b/e2e/playwright/tsconfig.json
@@ -0,0 +1,7 @@
+{
+  "compilerOptions": {
+    "types": [
+      "node"
+    ]
+  },
+}
diff --git a/e2e/scripts/playwright.sh b/e2e/scripts/playwright.sh
index 1fbcd93ef..744286477 100755
--- a/e2e/scripts/playwright.sh
+++ b/e2e/scripts/playwright.sh
@@ -15,6 +15,8 @@ main() {
     export DR_AWS_S3_PREFIX="$5"
     export DR_AWS_ACCESS_KEY_ID="$6"
     export DR_AWS_SECRET_ACCESS_KEY="$7"
+  elif [ "$test_name" == "deploy-upgrade" ]; then
+    export APP_UPGRADE_VERSION="$1"
   fi
 
   # if we have a second argument and the first points to the

From 1dc044ebd9e6f19c731e74ab2841753493e756b5 Mon Sep 17 00:00:00 2001
From: Ethan Mosbaugh <ethan@replicated.com>
Date: Tue, 9 Jul 2024 15:58:08 -0700
Subject: [PATCH 50/96] upgrade test

---
 .github/actions/e2e/action.yml            |  1 -
 e2e/cluster/cluster.go                    | 19 +++++++++++--------
 e2e/install_test.go                       |  9 +++------
 e2e/playwright/tests/shared/deploy-app.ts |  1 +
 4 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/.github/actions/e2e/action.yml b/.github/actions/e2e/action.yml
index 5fb08fef0..332e21920 100644
--- a/.github/actions/e2e/action.yml
+++ b/.github/actions/e2e/action.yml
@@ -104,7 +104,6 @@ runs:
       export DR_AWS_S3_PREFIX_AIRGAP=${{ inputs.test-name }}-${{ github.run_id }}-${{ github.run_attempt }}-airgap
       export DR_AWS_ACCESS_KEY_ID=${{ inputs.dr-aws-access-key-id }}
       export DR_AWS_SECRET_ACCESS_KEY=${{ inputs.dr-aws-secret-access-key }}
-      export APP_UPGRADE_VERSION=appver-dev-${SHORT_SHA}-upgrade
       make e2e-test TEST_NAME=${{ inputs.test-name }}
   - name: Upload Host Support Bundle
     uses: actions/upload-artifact@v4
diff --git a/e2e/cluster/cluster.go b/e2e/cluster/cluster.go
index c479121f8..a25d1b6b0 100644
--- a/e2e/cluster/cluster.go
+++ b/e2e/cluster/cluster.go
@@ -92,6 +92,7 @@ type Output struct {
 
 // Destroy destroys a cluster pointed by the id property inside the output.
 func (o *Output) Destroy() {
+	o.T.Logf("Destroying cluster %s", o.id)
 	client, err := lxd.ConnectLXDUnix(lxdSocket, nil)
 	if err != nil {
 		o.T.Fatalf("Failed to connect to LXD: %v", err)
@@ -201,23 +202,25 @@ func NewTestCluster(in *Input) *Output {
 
 	in.id = uuid.New().String()[:5]
 	in.network = <-networkaddr
+
+	out := &Output{
+		T:       in.T,
+		network: in.network,
+		id:      in.id,
+	}
+	out.T.Cleanup(out.Destroy)
+
 	PullImage(in)
 	CreateProfile(in)
 	CreateNetworks(in)
-	nodes := CreateNodes(in)
-	for _, node := range nodes {
+	out.Nodes = CreateNodes(in)
+	for _, node := range out.Nodes {
 		CopyFilesToNode(in, node)
 		CopyDirsToNode(in, node)
 		if in.CreateRegularUser {
 			CreateRegularUser(in, node)
 		}
 	}
-	out := &Output{
-		T:       in.T,
-		Nodes:   nodes,
-		network: in.network,
-		id:      in.id,
-	}
 	if in.WithProxy {
 		out.Proxy = CreateProxy(in)
 		CopyDirsToNode(in, out.Proxy)
diff --git a/e2e/install_test.go b/e2e/install_test.go
index 380a10754..fdd3d2419 100644
--- a/e2e/install_test.go
+++ b/e2e/install_test.go
@@ -577,7 +577,7 @@ func TestSingleNodeUpgrade(t *testing.T) {
 	t.Parallel()
 
 	requiredEnvVars := []string{
-		"APP_UPGRADE_VERSION",
+		"SHORT_SHA",
 	}
 	for _, envVar := range requiredEnvVars {
 		if os.Getenv(envVar) == "" {
@@ -613,10 +613,8 @@ func TestSingleNodeUpgrade(t *testing.T) {
 		t.Fatalf("fail to check installation state: %v", err)
 	}
 
-	testArgs := []string{}
-	for _, envVar := range requiredEnvVars {
-		testArgs = append(testArgs, os.Getenv(envVar))
-	}
+	appUpgradeVersion := fmt.Sprintf("appver-dev-%s-upgrade", os.Getenv("SHORT_SHA"))
+	testArgs := []string{appUpgradeVersion}
 
 	t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339))
 	if _, _, err := runPlaywrightTest(t, tc, "deploy-upgrade", testArgs...); err != nil {
@@ -1712,5 +1710,4 @@ func cleanupCluster(t *testing.T, tc *cluster.Output) {
 		generateAndCopySupportBundle(t, tc)
 		copyPlaywrightReport(t, tc)
 	}
-	tc.Destroy()
 }
diff --git a/e2e/playwright/tests/shared/deploy-app.ts b/e2e/playwright/tests/shared/deploy-app.ts
index b28c8f9d8..72ff20eab 100644
--- a/e2e/playwright/tests/shared/deploy-app.ts
+++ b/e2e/playwright/tests/shared/deploy-app.ts
@@ -7,6 +7,7 @@ export const deployApp = async (page, expect) => {
   await page.locator('input[type="password"]').click();
   await page.locator('input[type="password"]').fill('password');
   await page.getByRole('button', { name: 'Continue' }).click();
+  await page.getByRole('button', { name: 'Deploy' }).click();
   await expect(page.locator('#app')).toContainText('Currently deployed version', { timeout: 90000 });
   await expect(page.locator('#app')).toContainText('Ready', { timeout: 30000 });
   await expect(page.locator('#app')).toContainText('Up to date');

From 1eb72d987b1275bb820ee326916ecb91788e952b Mon Sep 17 00:00:00 2001
From: Ethan Mosbaugh <ethan@replicated.com>
Date: Tue, 9 Jul 2024 16:42:28 -0700
Subject: [PATCH 51/96] upgrade test

---
 e2e/playwright/tests/deploy-upgrade/test.spec.ts | 6 +++++-
 e2e/scripts/playwright.sh                        | 2 +-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/e2e/playwright/tests/deploy-upgrade/test.spec.ts b/e2e/playwright/tests/deploy-upgrade/test.spec.ts
index e30a0ecef..a7f0d879f 100644
--- a/e2e/playwright/tests/deploy-upgrade/test.spec.ts
+++ b/e2e/playwright/tests/deploy-upgrade/test.spec.ts
@@ -5,7 +5,8 @@ test('deploy upgrade', async ({ page }) => {
   test.setTimeout(15 * 60 * 1000); // 15 minutes
   await login(page);
   await page.getByRole('link', { name: 'Version history', exact: true }).click();
-  const rowLocator = '../../..'; // '//./ancestor::div[@class="available-update-row"]'
+  const rowLocator = '../../..';
+  // const rowLocator = '//./ancestor::div[@class="available-update-row"]';
   await page.getByText(process.env.APP_UPGRADE_VERSION, { exact: true })
     .locator(rowLocator)
     .getByRole('button', { name: 'Deploy', exact: true }).click();
@@ -18,4 +19,7 @@ test('deploy upgrade', async ({ page }) => {
   await expect(page.locator('.Modal-body').getByText('Cluster update in progress')).not.toBeVisible({ timeout: 60 * 1000 });
   await page.getByRole('link', { name: 'Dashboard', exact: true }).click();
   await expect(page.locator('.VersionCard-content--wrapper')).toContainText(process.env.APP_UPGRADE_VERSION);
+  await expect(page.locator('#app')).toContainText('Currently deployed version', { timeout: 90000 });
+  await expect(page.locator('#app')).toContainText('Ready', { timeout: 30000 });
+  await expect(page.locator('#app')).toContainText('Up to date');
 });
diff --git a/e2e/scripts/playwright.sh b/e2e/scripts/playwright.sh
index 744286477..03ecbd968 100755
--- a/e2e/scripts/playwright.sh
+++ b/e2e/scripts/playwright.sh
@@ -16,7 +16,7 @@ main() {
     export DR_AWS_ACCESS_KEY_ID="$6"
     export DR_AWS_SECRET_ACCESS_KEY="$7"
   elif [ "$test_name" == "deploy-upgrade" ]; then
-    export APP_UPGRADE_VERSION="$1"
+    export APP_UPGRADE_VERSION="$2"
   fi
 
   # if we have a second argument and the first points to the

From 1ab5725a059ca151e70609f43a38e8a5a9e5f4c3 Mon Sep 17 00:00:00 2001
From: Ethan Mosbaugh <ethan@replicated.com>
Date: Tue, 9 Jul 2024 18:29:17 -0700
Subject: [PATCH 52/96] upgrade test

---
 e2e/install_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/e2e/install_test.go b/e2e/install_test.go
index fdd3d2419..6c2ed0b10 100644
--- a/e2e/install_test.go
+++ b/e2e/install_test.go
@@ -613,7 +613,7 @@ func TestSingleNodeUpgrade(t *testing.T) {
 		t.Fatalf("fail to check installation state: %v", err)
 	}
 
-	appUpgradeVersion := fmt.Sprintf("appver-dev-%s-upgrade", os.Getenv("SHORT_SHA"))
+	appUpgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA"))
 	testArgs := []string{appUpgradeVersion}
 
 	t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339))

From de091de85045d0bf4866fc957e400b3259b7fd41 Mon Sep 17 00:00:00 2001
From: Ethan Mosbaugh <ethan@replicated.com>
Date: Tue, 9 Jul 2024 18:54:54 -0700
Subject: [PATCH 53/96] upgrade test

---
 e2e/playwright/tests/deploy-upgrade/test.spec.ts | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/e2e/playwright/tests/deploy-upgrade/test.spec.ts b/e2e/playwright/tests/deploy-upgrade/test.spec.ts
index a7f0d879f..9b12416f1 100644
--- a/e2e/playwright/tests/deploy-upgrade/test.spec.ts
+++ b/e2e/playwright/tests/deploy-upgrade/test.spec.ts
@@ -16,10 +16,10 @@ test('deploy upgrade', async ({ page }) => {
   await iframe.getByRole('button', { name: 'Next: Confirm and deploy', exact: true }).click({ timeout: 10 * 1000 });
   await iframe.getByRole('button', { name: 'Deploy', exact: true }).click();
   await expect(page.locator('.Modal-body').getByText('Cluster update in progress')).toBeVisible();
-  await expect(page.locator('.Modal-body').getByText('Cluster update in progress')).not.toBeVisible({ timeout: 60 * 1000 });
+  await expect(page.locator('.Modal-body').getByText('Cluster update in progress')).not.toBeVisible({ timeout: 5 * 60 * 1000 });
   await page.getByRole('link', { name: 'Dashboard', exact: true }).click();
   await expect(page.locator('.VersionCard-content--wrapper')).toContainText(process.env.APP_UPGRADE_VERSION);
-  await expect(page.locator('#app')).toContainText('Currently deployed version', { timeout: 90000 });
-  await expect(page.locator('#app')).toContainText('Ready', { timeout: 30000 });
+  await expect(page.locator('#app')).toContainText('Currently deployed version', { timeout: 2 * 60 * 1000 });
+  await expect(page.locator('#app')).toContainText('Ready', { timeout: 30 * 1000 });
   await expect(page.locator('#app')).toContainText('Up to date');
 });

From 4ae28e770aedce946716fb17253149ffb637fb11 Mon Sep 17 00:00:00 2001
From: Ethan Mosbaugh <ethan@replicated.com>
Date: Tue, 9 Jul 2024 20:06:08 -0700
Subject: [PATCH 54/96] upgrade test

---
 e2e/install_test.go | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/e2e/install_test.go b/e2e/install_test.go
index 6c2ed0b10..f4fed54c8 100644
--- a/e2e/install_test.go
+++ b/e2e/install_test.go
@@ -621,6 +621,12 @@ func TestSingleNodeUpgrade(t *testing.T) {
 		t.Fatalf("fail to run playwright test deploy-app: %v", err)
 	}
 
+	t.Logf("%s: checking installation state", time.Now().Format(time.RFC3339))
+	line = []string{"check-installation-state.sh", appUpgradeVersion}
+	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
+		t.Fatalf("fail to check installation state: %v", err)
+	}
+
 	t.Logf("%s: test complete", time.Now().Format(time.RFC3339))
 }
 

From 4a0f4d2ac25fdd3b1886e55b8693f5f4755851e9 Mon Sep 17 00:00:00 2001
From: Ethan Mosbaugh <ethan@replicated.com>
Date: Tue, 9 Jul 2024 20:07:48 -0700
Subject: [PATCH 55/96] upgrade test

---
 e2e/install_test.go | 114 ++++++++++++++++++++++----------------------
 1 file changed, 57 insertions(+), 57 deletions(-)

diff --git a/e2e/install_test.go b/e2e/install_test.go
index f4fed54c8..3ab85d0bc 100644
--- a/e2e/install_test.go
+++ b/e2e/install_test.go
@@ -235,6 +235,63 @@ func TestSingleNodeInstallationCentos9Stream(t *testing.T) {
 	t.Logf("%s: test complete", time.Now().Format(time.RFC3339))
 }
 
+func TestSingleNodeUpgrade(t *testing.T) {
+	t.Parallel()
+
+	requiredEnvVars := []string{
+		"SHORT_SHA",
+	}
+	for _, envVar := range requiredEnvVars {
+		if os.Getenv(envVar) == "" {
+			t.Fatalf("missing required environment variable: %s", envVar)
+		}
+	}
+
+	tc := cluster.NewTestCluster(&cluster.Input{
+		T:                   t,
+		Nodes:               1,
+		Image:               "ubuntu/jammy",
+		LicensePath:         "license.yaml",
+		EmbeddedClusterPath: "../output/bin/embedded-cluster",
+	})
+	defer cleanupCluster(t, tc)
+
+	t.Logf("%s: installing embedded-cluster on node 0", time.Now().Format(time.RFC3339))
+	line := []string{"single-node-install.sh", "ui"}
+	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
+		t.Fatalf("fail to install embedded-cluster on node %s: %v", tc.Nodes[0], err)
+	}
+
+	if err := setupPlaywright(t, tc); err != nil {
+		t.Fatalf("fail to setup playwright: %v", err)
+	}
+	if _, _, err := runPlaywrightTest(t, tc, "deploy-app"); err != nil {
+		t.Fatalf("fail to run playwright test deploy-app: %v", err)
+	}
+
+	t.Logf("%s: checking installation state", time.Now().Format(time.RFC3339))
+	line = []string{"check-installation-state.sh", os.Getenv("SHORT_SHA")}
+	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
+		t.Fatalf("fail to check installation state: %v", err)
+	}
+
+	appUpgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA"))
+	testArgs := []string{appUpgradeVersion}
+
+	t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339))
+	if _, _, err := runPlaywrightTest(t, tc, "deploy-upgrade", testArgs...); err != nil {
+		t.Fatalf("fail to run playwright test deploy-app: %v", err)
+	}
+
+	t.Logf("%s: checking installation state after upgrade", time.Now().Format(time.RFC3339))
+	line = []string{"check-postupgrade-state.sh"}
+	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
+		t.Fatalf("fail to check postupgrade state: %v", err)
+	}
+
+	t.Logf("%s: test complete", time.Now().Format(time.RFC3339))
+}
+
 func TestHostPreflight(t *testing.T) {
 	t.Parallel()
 	tc := cluster.NewTestCluster(&cluster.Input{
@@ -573,63 +630,6 @@ func TestOldVersionUpgrade(t *testing.T) {
 	t.Logf("%s: test complete", time.Now().Format(time.RFC3339))
 }
 
-func TestSingleNodeUpgrade(t *testing.T) {
-	t.Parallel()
-
-	requiredEnvVars := []string{
-		"SHORT_SHA",
-	}
-	for _, envVar := range requiredEnvVars {
-		if os.Getenv(envVar) == "" {
-			t.Fatalf("missing required environment variable: %s", envVar)
-		}
-	}
-
-	tc := cluster.NewTestCluster(&cluster.Input{
-		T:                   t,
-		Nodes:               1,
-		Image:               "ubuntu/jammy",
-		LicensePath:         "license.yaml",
-		EmbeddedClusterPath: "../output/bin/embedded-cluster",
-	})
-	defer cleanupCluster(t, tc)
-
-	t.Logf("%s: installing embedded-cluster on node 0", time.Now().Format(time.RFC3339))
-	line := []string{"single-node-install.sh", "ui"}
-	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
-		t.Fatalf("fail to install embedded-cluster on node %s: %v", tc.Nodes[0], err)
-	}
-
-	if err := setupPlaywright(t, tc); err != nil {
-		t.Fatalf("fail to setup playwright: %v", err)
-	}
-	if _, _, err := runPlaywrightTest(t, tc, "deploy-app"); err != nil {
-		t.Fatalf("fail to run playwright test deploy-app: %v", err)
-	}
-
-	t.Logf("%s: checking installation state", time.Now().Format(time.RFC3339))
-	line = []string{"check-installation-state.sh", os.Getenv("SHORT_SHA")}
-	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
-		t.Fatalf("fail to check installation state: %v", err)
-	}
-
-	appUpgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA"))
-	testArgs := []string{appUpgradeVersion}
-
-	t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339))
-	if _, _, err := runPlaywrightTest(t, tc, "deploy-upgrade", testArgs...); err != nil {
-		t.Fatalf("fail to run playwright test deploy-app: %v", err)
-	}
-
-	t.Logf("%s: checking installation state", time.Now().Format(time.RFC3339))
-	line = []string{"check-installation-state.sh", appUpgradeVersion}
-	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
-		t.Fatalf("fail to check installation state: %v", err)
-	}
-
-	t.Logf("%s: test complete", time.Now().Format(time.RFC3339))
-}
-
 func TestSingleNodeAirgapUpgrade(t *testing.T) {
 	t.Parallel()
 

From 09da2d112145f205ec8f469d7d49166bbe448c92 Mon Sep 17 00:00:00 2001
From: Ethan Mosbaugh <ethan@replicated.com>
Date: Tue, 9 Jul 2024 22:20:11 -0700
Subject: [PATCH 56/96] upgrade test

---
 .github/workflows/pull-request.yaml |  89 ++++++++-------
 .github/workflows/release-dev.yaml  |   1 -
 e2e/install_test.go                 | 163 ++++++++++++++--------------
 e2e/restore_test.go                 |  37 ++-----
 e2e/utils.go                        |   9 ++
 5 files changed, 144 insertions(+), 155 deletions(-)

diff --git a/.github/workflows/pull-request.yaml b/.github/workflows/pull-request.yaml
index d30eabd8d..d389d7742 100644
--- a/.github/workflows/pull-request.yaml
+++ b/.github/workflows/pull-request.yaml
@@ -199,51 +199,50 @@ jobs:
       fail-fast: false
       matrix:
         test:
-          # - TestSingleNodeInstallation
-          # - TestSingleNodeInstallationAlmaLinux8
-          # - TestSingleNodeInstallationDebian11
-          # - TestSingleNodeInstallationDebian12
-          # - TestSingleNodeInstallationCentos9Stream
-          - TestSingleNodeUpgrade
-        #   - TestVersion
-        #   - TestHostPreflight
-        #   - TestUnsupportedOverrides
-        #   - TestMultiNodeInstallation
-        #   - TestMultiNodeReset
-        #   - TestCommandsRequireSudo
-        #   - TestInstallWithoutEmbed
-        #   - TestInstallFromReplicatedApp
-        #   - TestResetAndReinstall
-        #   - TestResetAndReinstallAirgap
-        #   - TestCollectSupportBundle
-        #   - TestOldVersionUpgrade
-        #   - TestMaterialize
-        #   - TestLocalArtifactMirror
-        #   - TestSingleNodeAirgapUpgrade
-        #   - TestSingleNodeAirgapUpgradeCustomCIDR
-        #   - TestInstallSnapshotFromReplicatedApp
-        #   - TestMultiNodeAirgapUpgrade
-        #   - TestSingleNodeDisasterRecovery
-        #   - TestSingleNodeDisasterRecoveryWithProxy
-        #   - TestSingleNodeResumeDisasterRecovery
-        #   - TestSingleNodeAirgapDisasterRecovery
-        #   - TestMultiNodeHAInstallation
-        #   - TestMultiNodeAirgapHAInstallation
-        #   - TestMultiNodeAirgapUpgradeSameK0s
-        #   - TestProxiedEnvironment
-        #   - TestMultiNodeHADisasterRecovery
-        #   - TestMultiNodeAirgapHADisasterRecovery
-        #   - TestCustomCIDR
-        #   - TestProxiedCustomCIDR
-        include: []
-        #   - test: TestMultiNodeAirgapUpgrade
-        #     runner: embedded-cluster
-        #   - test: TestMultiNodeAirgapHAInstallation
-        #     runner: embedded-cluster
-        #   - test: TestMultiNodeAirgapHADisasterRecovery
-        #     runner: embedded-cluster
-        #   - test: TestSingleNodeAirgapDisasterRecovery
-        #     runner: embedded-cluster
+          - TestSingleNodeInstallation
+          - TestSingleNodeInstallationAlmaLinux8
+          - TestSingleNodeInstallationDebian11
+          - TestSingleNodeInstallationDebian12
+          - TestSingleNodeInstallationCentos9Stream
+          - TestVersion
+          - TestHostPreflight
+          - TestUnsupportedOverrides
+          - TestMultiNodeInstallation
+          - TestMultiNodeReset
+          - TestCommandsRequireSudo
+          - TestInstallWithoutEmbed
+          - TestInstallFromReplicatedApp
+          - TestResetAndReinstall
+          - TestResetAndReinstallAirgap
+          - TestCollectSupportBundle
+          - TestOldVersionUpgrade
+          - TestMaterialize
+          - TestLocalArtifactMirror
+          - TestSingleNodeAirgapUpgrade
+          - TestSingleNodeAirgapUpgradeCustomCIDR
+          - TestInstallSnapshotFromReplicatedApp
+          - TestMultiNodeAirgapUpgrade
+          - TestSingleNodeDisasterRecovery
+          - TestSingleNodeDisasterRecoveryWithProxy
+          - TestSingleNodeResumeDisasterRecovery
+          - TestSingleNodeAirgapDisasterRecovery
+          - TestMultiNodeHAInstallation
+          - TestMultiNodeAirgapHAInstallation
+          - TestMultiNodeAirgapUpgradeSameK0s
+          - TestProxiedEnvironment
+          - TestMultiNodeHADisasterRecovery
+          - TestMultiNodeAirgapHADisasterRecovery
+          - TestCustomCIDR
+          - TestProxiedCustomCIDR
+        include:
+          - test: TestMultiNodeAirgapUpgrade
+            runner: embedded-cluster
+          - test: TestMultiNodeAirgapHAInstallation
+            runner: embedded-cluster
+          - test: TestMultiNodeAirgapHADisasterRecovery
+            runner: embedded-cluster
+          - test: TestSingleNodeAirgapDisasterRecovery
+            runner: embedded-cluster
     steps:
     - name: Checkout
       uses: actions/checkout@v4
diff --git a/.github/workflows/release-dev.yaml b/.github/workflows/release-dev.yaml
index 080838758..bfd939c9d 100644
--- a/.github/workflows/release-dev.yaml
+++ b/.github/workflows/release-dev.yaml
@@ -149,7 +149,6 @@ jobs:
           - TestSingleNodeInstallationDebian11
           - TestSingleNodeInstallationDebian12
           - TestSingleNodeInstallationCentos9Stream
-          - TestSingleNodeUpgrade
           - TestVersion
           - TestHostPreflight
           - TestUnsupportedOverrides
diff --git a/e2e/install_test.go b/e2e/install_test.go
index 3ab85d0bc..1b73dd31c 100644
--- a/e2e/install_test.go
+++ b/e2e/install_test.go
@@ -14,6 +14,9 @@ import (
 
 func TestSingleNodeInstallation(t *testing.T) {
 	t.Parallel()
+
+	RequireEnvVars(t, []string{"SHORT_SHA"})
+
 	tc := cluster.NewTestCluster(&cluster.Input{
 		T:                   t,
 		Nodes:               1,
@@ -36,10 +39,12 @@ func TestSingleNodeInstallation(t *testing.T) {
 		t.Fatalf("fail to run playwright test deploy-app: %v", err)
 	}
 
-	t.Logf("%s: running kots upstream upgrade", time.Now().Format(time.RFC3339))
-	line = []string{"kots-upstream-upgrade.sh", os.Getenv("SHORT_SHA")}
-	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
-		t.Fatalf("fail to run kots upstream upgrade: %v", err)
+	appUpgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA"))
+	testArgs := []string{appUpgradeVersion}
+
+	t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339))
+	if _, _, err := runPlaywrightTest(t, tc, "deploy-upgrade", testArgs...); err != nil {
+		t.Fatalf("fail to run playwright test deploy-app: %v", err)
 	}
 
 	t.Logf("%s: checking installation state after upgrade", time.Now().Format(time.RFC3339))
@@ -53,6 +58,9 @@ func TestSingleNodeInstallation(t *testing.T) {
 
 func TestSingleNodeInstallationAlmaLinux8(t *testing.T) {
 	t.Parallel()
+
+	RequireEnvVars(t, []string{"SHORT_SHA"})
+
 	tc := cluster.NewTestCluster(&cluster.Input{
 		T:                   t,
 		Nodes:               1,
@@ -80,10 +88,12 @@ func TestSingleNodeInstallationAlmaLinux8(t *testing.T) {
 		t.Fatalf("fail to check installation state: %v", err)
 	}
 
-	t.Logf("%s: running kots upstream upgrade", time.Now().Format(time.RFC3339))
-	line = []string{"kots-upstream-upgrade.sh", os.Getenv("SHORT_SHA")}
-	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
-		t.Fatalf("fail to run kots upstream upgrade: %v", err)
+	appUpgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA"))
+	testArgs := []string{appUpgradeVersion}
+
+	t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339))
+	if _, _, err := runPlaywrightTest(t, tc, "deploy-upgrade", testArgs...); err != nil {
+		t.Fatalf("fail to run playwright test deploy-app: %v", err)
 	}
 
 	t.Logf("%s: checking installation state after upgrade", time.Now().Format(time.RFC3339))
@@ -97,6 +107,9 @@ func TestSingleNodeInstallationAlmaLinux8(t *testing.T) {
 
 func TestSingleNodeInstallationDebian12(t *testing.T) {
 	t.Parallel()
+
+	RequireEnvVars(t, []string{"SHORT_SHA"})
+
 	tc := cluster.NewTestCluster(&cluster.Input{
 		T:                   t,
 		Nodes:               1,
@@ -128,10 +141,12 @@ func TestSingleNodeInstallationDebian12(t *testing.T) {
 		t.Fatalf("fail to check installation state: %v", err)
 	}
 
-	t.Logf("%s: running kots upstream upgrade", time.Now().Format(time.RFC3339))
-	line = []string{"kots-upstream-upgrade.sh", os.Getenv("SHORT_SHA")}
-	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
-		t.Fatalf("fail to run kots upstream upgrade: %v", err)
+	appUpgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA"))
+	testArgs := []string{appUpgradeVersion}
+
+	t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339))
+	if _, _, err := runPlaywrightTest(t, tc, "deploy-upgrade", testArgs...); err != nil {
+		t.Fatalf("fail to run playwright test deploy-app: %v", err)
 	}
 
 	t.Logf("%s: checking installation state after upgrade", time.Now().Format(time.RFC3339))
@@ -145,6 +160,9 @@ func TestSingleNodeInstallationDebian12(t *testing.T) {
 
 func TestSingleNodeInstallationDebian11(t *testing.T) {
 	t.Parallel()
+
+	RequireEnvVars(t, []string{"SHORT_SHA"})
+
 	tc := cluster.NewTestCluster(&cluster.Input{
 		T:                   t,
 		Nodes:               1,
@@ -176,10 +194,12 @@ func TestSingleNodeInstallationDebian11(t *testing.T) {
 		t.Fatalf("fail to check installation state: %v", err)
 	}
 
-	t.Logf("%s: running kots upstream upgrade", time.Now().Format(time.RFC3339))
-	line = []string{"kots-upstream-upgrade.sh", os.Getenv("SHORT_SHA")}
-	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
-		t.Fatalf("fail to run kots upstream upgrade: %v", err)
+	appUpgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA"))
+	testArgs := []string{appUpgradeVersion}
+
+	t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339))
+	if _, _, err := runPlaywrightTest(t, tc, "deploy-upgrade", testArgs...); err != nil {
+		t.Fatalf("fail to run playwright test deploy-app: %v", err)
 	}
 
 	t.Logf("%s: checking installation state after upgrade", time.Now().Format(time.RFC3339))
@@ -193,6 +213,9 @@ func TestSingleNodeInstallationDebian11(t *testing.T) {
 
 func TestSingleNodeInstallationCentos9Stream(t *testing.T) {
 	t.Parallel()
+
+	RequireEnvVars(t, []string{"SHORT_SHA"})
+
 	tc := cluster.NewTestCluster(&cluster.Input{
 		T:                   t,
 		Nodes:               1,
@@ -220,61 +243,6 @@ func TestSingleNodeInstallationCentos9Stream(t *testing.T) {
 		t.Fatalf("fail to check installation state: %v", err)
 	}
 
-	t.Logf("%s: running kots upstream upgrade", time.Now().Format(time.RFC3339))
-	line = []string{"kots-upstream-upgrade.sh", os.Getenv("SHORT_SHA")}
-	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
-		t.Fatalf("fail to run kots upstream upgrade: %v", err)
-	}
-
-	t.Logf("%s: checking installation state after upgrade", time.Now().Format(time.RFC3339))
-	line = []string{"check-postupgrade-state.sh"}
-	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
-		t.Fatalf("fail to check postupgrade state: %v", err)
-	}
-
-	t.Logf("%s: test complete", time.Now().Format(time.RFC3339))
-}
-
-func TestSingleNodeUpgrade(t *testing.T) {
-	t.Parallel()
-
-	requiredEnvVars := []string{
-		"SHORT_SHA",
-	}
-	for _, envVar := range requiredEnvVars {
-		if os.Getenv(envVar) == "" {
-			t.Fatalf("missing required environment variable: %s", envVar)
-		}
-	}
-
-	tc := cluster.NewTestCluster(&cluster.Input{
-		T:                   t,
-		Nodes:               1,
-		Image:               "ubuntu/jammy",
-		LicensePath:         "license.yaml",
-		EmbeddedClusterPath: "../output/bin/embedded-cluster",
-	})
-	defer cleanupCluster(t, tc)
-
-	t.Logf("%s: installing embedded-cluster on node 0", time.Now().Format(time.RFC3339))
-	line := []string{"single-node-install.sh", "ui"}
-	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
-		t.Fatalf("fail to install embedded-cluster on node %s: %v", tc.Nodes[0], err)
-	}
-
-	if err := setupPlaywright(t, tc); err != nil {
-		t.Fatalf("fail to setup playwright: %v", err)
-	}
-	if _, _, err := runPlaywrightTest(t, tc, "deploy-app"); err != nil {
-		t.Fatalf("fail to run playwright test deploy-app: %v", err)
-	}
-
-	t.Logf("%s: checking installation state", time.Now().Format(time.RFC3339))
-	line = []string{"check-installation-state.sh", os.Getenv("SHORT_SHA")}
-	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
-		t.Fatalf("fail to check installation state: %v", err)
-	}
-
 	appUpgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA"))
 	testArgs := []string{appUpgradeVersion}
 
@@ -294,6 +262,9 @@ func TestSingleNodeUpgrade(t *testing.T) {
 
 func TestHostPreflight(t *testing.T) {
 	t.Parallel()
+
+	RequireEnvVars(t, []string{"SHORT_SHA"})
+
 	tc := cluster.NewTestCluster(&cluster.Input{
 		T:                                 t,
 		Nodes:                             1,
@@ -410,6 +381,9 @@ func TestMultiNodeInstallation(t *testing.T) {
 
 func TestInstallWithoutEmbed(t *testing.T) {
 	t.Parallel()
+
+	RequireEnvVars(t, []string{"SHORT_SHA"})
+
 	tc := cluster.NewTestCluster(&cluster.Input{
 		T:                   t,
 		Nodes:               1,
@@ -430,6 +404,9 @@ func TestInstallWithoutEmbed(t *testing.T) {
 
 func TestInstallFromReplicatedApp(t *testing.T) {
 	t.Parallel()
+
+	RequireEnvVars(t, []string{"SHORT_SHA"})
+
 	tc := cluster.NewTestCluster(&cluster.Input{
 		T:     t,
 		Nodes: 1,
@@ -455,10 +432,12 @@ func TestInstallFromReplicatedApp(t *testing.T) {
 		t.Fatalf("fail to check installation state: %v", err)
 	}
 
-	t.Logf("%s: running kots upstream upgrade", time.Now().Format(time.RFC3339))
-	line = []string{"kots-upstream-upgrade.sh", os.Getenv("SHORT_SHA")}
-	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
-		t.Fatalf("fail to run kots upstream upgrade: %v", err)
+	appUpgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA"))
+	testArgs := []string{appUpgradeVersion}
+
+	t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339))
+	if _, _, err := runPlaywrightTest(t, tc, "deploy-upgrade", testArgs...); err != nil {
+		t.Fatalf("fail to run playwright test deploy-app: %v", err)
 	}
 
 	t.Logf("%s: checking installation state after upgrade", time.Now().Format(time.RFC3339))
@@ -472,6 +451,9 @@ func TestInstallFromReplicatedApp(t *testing.T) {
 
 func TestResetAndReinstall(t *testing.T) {
 	t.Parallel()
+
+	RequireEnvVars(t, []string{"SHORT_SHA"})
+
 	tc := cluster.NewTestCluster(&cluster.Input{
 		T:                   t,
 		Nodes:               1,
@@ -517,6 +499,8 @@ func TestResetAndReinstall(t *testing.T) {
 func TestResetAndReinstallAirgap(t *testing.T) {
 	t.Parallel()
 
+	RequireEnvVars(t, []string{"SHORT_SHA"})
+
 	t.Logf("%s: downloading airgap file", time.Now().Format(time.RFC3339))
 	// download airgap bundle
 	airgapURL := fmt.Sprintf("https://staging.replicated.app/embedded/embedded-cluster-smoke-test-staging-app/ci-airgap/appver-%s?airgap=true", os.Getenv("SHORT_SHA"))
@@ -590,6 +574,9 @@ func TestResetAndReinstallAirgap(t *testing.T) {
 
 func TestOldVersionUpgrade(t *testing.T) {
 	t.Parallel()
+
+	RequireEnvVars(t, []string{"SHORT_SHA"})
+
 	tc := cluster.NewTestCluster(&cluster.Input{
 		T:     t,
 		Nodes: 1,
@@ -633,6 +620,8 @@ func TestOldVersionUpgrade(t *testing.T) {
 func TestSingleNodeAirgapUpgrade(t *testing.T) {
 	t.Parallel()
 
+	RequireEnvVars(t, []string{"SHORT_SHA"})
+
 	t.Logf("%s: downloading airgap files", time.Now().Format(time.RFC3339))
 	airgapInstallBundlePath := "/tmp/airgap-install-bundle.tar.gz"
 	airgapUpgradeBundlePath := "/tmp/airgap-upgrade-bundle.tar.gz"
@@ -737,6 +726,8 @@ func TestSingleNodeAirgapUpgrade(t *testing.T) {
 func TestSingleNodeAirgapUpgradeCustomCIDR(t *testing.T) {
 	t.Parallel()
 
+	RequireEnvVars(t, []string{"SHORT_SHA"})
+
 	t.Logf("%s: downloading airgap files", time.Now().Format(time.RFC3339))
 	airgapInstallBundlePath := "/tmp/airgap-install-bundle.tar.gz"
 	airgapUpgradeBundlePath := "/tmp/airgap-upgrade-bundle.tar.gz"
@@ -851,6 +842,8 @@ func TestSingleNodeAirgapUpgradeCustomCIDR(t *testing.T) {
 func TestMultiNodeAirgapUpgradeSameK0s(t *testing.T) {
 	t.Parallel()
 
+	RequireEnvVars(t, []string{"SHORT_SHA"})
+
 	t.Logf("%s: downloading airgap files", time.Now().Format(time.RFC3339))
 	airgapInstallBundlePath := "/tmp/airgap-install-bundle.tar.gz"
 	airgapUpgradeBundlePath := "/tmp/airgap-upgrade-bundle.tar.gz"
@@ -1017,6 +1010,8 @@ func TestMultiNodeAirgapUpgradeSameK0s(t *testing.T) {
 func TestMultiNodeAirgapUpgrade(t *testing.T) {
 	t.Parallel()
 
+	RequireEnvVars(t, []string{"SHORT_SHA"})
+
 	t.Logf("%s: downloading airgap files", time.Now().Format(time.RFC3339))
 	airgapInstallBundlePath := "/tmp/airgap-install-bundle.tar.gz"
 	airgapUpgradeBundlePath := "/tmp/airgap-upgrade-bundle.tar.gz"
@@ -1290,6 +1285,8 @@ func TestMultiNodeHAInstallation(t *testing.T) {
 func TestMultiNodeAirgapHAInstallation(t *testing.T) {
 	t.Parallel()
 
+	RequireEnvVars(t, []string{"SHORT_SHA"})
+
 	t.Logf("%s: downloading airgap file", time.Now().Format(time.RFC3339))
 	airgapInstallBundlePath := "/tmp/airgap-install-bundle.tar.gz"
 	downloadAirgapBundle(t, fmt.Sprintf("appver-%s", os.Getenv("SHORT_SHA")), airgapInstallBundlePath, os.Getenv("AIRGAP_LICENSE_ID"))
@@ -1448,6 +1445,9 @@ func TestMultiNodeAirgapHAInstallation(t *testing.T) {
 
 func TestInstallSnapshotFromReplicatedApp(t *testing.T) {
 	t.Parallel()
+
+	RequireEnvVars(t, []string{"SHORT_SHA"})
+
 	tc := cluster.NewTestCluster(&cluster.Input{
 		T:     t,
 		Nodes: 1,
@@ -1479,10 +1479,12 @@ func TestInstallSnapshotFromReplicatedApp(t *testing.T) {
 		t.Fatalf("fail to check velero state: %v", err)
 	}
 
-	t.Logf("%s: running kots upstream upgrade", time.Now().Format(time.RFC3339))
-	line = []string{"kots-upstream-upgrade.sh", os.Getenv("SHORT_SHA")}
-	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
-		t.Fatalf("fail to run kots upstream upgrade: %v", err)
+	appUpgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA"))
+	testArgs := []string{appUpgradeVersion}
+
+	t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339))
+	if _, _, err := runPlaywrightTest(t, tc, "deploy-upgrade", testArgs...); err != nil {
+		t.Fatalf("fail to run playwright test deploy-app: %v", err)
 	}
 
 	t.Logf("%s: checking installation state after upgrade", time.Now().Format(time.RFC3339))
@@ -1497,6 +1499,9 @@ func TestInstallSnapshotFromReplicatedApp(t *testing.T) {
 // TestCustomCIDR tests the installation with an alternate CIDR range
 func TestCustomCIDR(t *testing.T) {
 	t.Parallel()
+
+	RequireEnvVars(t, []string{"SHORT_SHA"})
+
 	tc := cluster.NewTestCluster(&cluster.Input{
 		T:                   t,
 		Nodes:               4,
diff --git a/e2e/restore_test.go b/e2e/restore_test.go
index e3b0d1953..a1a87092f 100644
--- a/e2e/restore_test.go
+++ b/e2e/restore_test.go
@@ -22,11 +22,7 @@ func TestSingleNodeDisasterRecovery(t *testing.T) {
 		"DR_AWS_ACCESS_KEY_ID",
 		"DR_AWS_SECRET_ACCESS_KEY",
 	}
-	for _, envVar := range requiredEnvVars {
-		if os.Getenv(envVar) == "" {
-			t.Fatalf("missing required environment variable: %s", envVar)
-		}
-	}
+	RequireEnvVars(t, requiredEnvVars)
 
 	testArgs := []string{}
 	for _, envVar := range requiredEnvVars {
@@ -93,11 +89,7 @@ func TestSingleNodeDisasterRecoveryWithProxy(t *testing.T) {
 		"DR_AWS_ACCESS_KEY_ID",
 		"DR_AWS_SECRET_ACCESS_KEY",
 	}
-	for _, envVar := range requiredEnvVars {
-		if os.Getenv(envVar) == "" {
-			t.Fatalf("missing required environment variable: %s", envVar)
-		}
-	}
+	RequireEnvVars(t, requiredEnvVars)
 
 	testArgs := []string{}
 	for _, envVar := range requiredEnvVars {
@@ -185,11 +177,7 @@ func TestSingleNodeResumeDisasterRecovery(t *testing.T) {
 		"DR_AWS_ACCESS_KEY_ID",
 		"DR_AWS_SECRET_ACCESS_KEY",
 	}
-	for _, envVar := range requiredEnvVars {
-		if os.Getenv(envVar) == "" {
-			t.Fatalf("missing required environment variable: %s", envVar)
-		}
-	}
+	RequireEnvVars(t, requiredEnvVars)
 
 	testArgs := []string{}
 	for _, envVar := range requiredEnvVars {
@@ -255,11 +243,8 @@ func TestSingleNodeAirgapDisasterRecovery(t *testing.T) {
 		"DR_AWS_ACCESS_KEY_ID",
 		"DR_AWS_SECRET_ACCESS_KEY",
 	}
-	for _, envVar := range requiredEnvVars {
-		if os.Getenv(envVar) == "" {
-			t.Fatalf("missing required environment variable: %s", envVar)
-		}
-	}
+	RequireEnvVars(t, requiredEnvVars)
+
 	testArgs := []string{}
 	for _, envVar := range requiredEnvVars {
 		testArgs = append(testArgs, os.Getenv(envVar))
@@ -417,11 +402,7 @@ func TestMultiNodeHADisasterRecovery(t *testing.T) {
 		"DR_AWS_ACCESS_KEY_ID",
 		"DR_AWS_SECRET_ACCESS_KEY",
 	}
-	for _, envVar := range requiredEnvVars {
-		if os.Getenv(envVar) == "" {
-			t.Fatalf("missing required environment variable: %s", envVar)
-		}
-	}
+	RequireEnvVars(t, requiredEnvVars)
 
 	testArgs := []string{}
 	for _, envVar := range requiredEnvVars {
@@ -608,11 +589,7 @@ func TestMultiNodeAirgapHADisasterRecovery(t *testing.T) {
 		"DR_AWS_ACCESS_KEY_ID",
 		"DR_AWS_SECRET_ACCESS_KEY",
 	}
-	for _, envVar := range requiredEnvVars {
-		if os.Getenv(envVar) == "" {
-			t.Fatalf("missing required environment variable: %s", envVar)
-		}
-	}
+	RequireEnvVars(t, requiredEnvVars)
 
 	testArgs := []string{}
 	for _, envVar := range requiredEnvVars {
diff --git a/e2e/utils.go b/e2e/utils.go
index 9180e53ce..57fc9de72 100644
--- a/e2e/utils.go
+++ b/e2e/utils.go
@@ -5,6 +5,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"os"
 	"regexp"
 	"strings"
 	"testing"
@@ -21,6 +22,14 @@ func (b *buffer) Close() error {
 	return nil
 }
 
+func RequireEnvVars(t *testing.T, envVars []string) {
+	for _, envVar := range envVars {
+		if os.Getenv(envVar) == "" {
+			t.Fatalf("missing required environment variable: %s", envVar)
+		}
+	}
+}
+
 type RunCommandOption func(cmd *cluster.Command)
 
 func WithEnv(env map[string]string) RunCommandOption {

From c7c30dc4319e7312cc13073e83c093bc78a7072f Mon Sep 17 00:00:00 2001
From: Ethan Mosbaugh <ethan@replicated.com>
Date: Wed, 10 Jul 2024 06:02:09 -0700
Subject: [PATCH 57/96] upgrade test

---
 e2e/install_test.go | 91 +++++++++++++++++++++++++++++++++++----------
 1 file changed, 72 insertions(+), 19 deletions(-)

diff --git a/e2e/install_test.go b/e2e/install_test.go
index 1b73dd31c..c6886e076 100644
--- a/e2e/install_test.go
+++ b/e2e/install_test.go
@@ -48,7 +48,7 @@ func TestSingleNodeInstallation(t *testing.T) {
 	}
 
 	t.Logf("%s: checking installation state after upgrade", time.Now().Format(time.RFC3339))
-	line = []string{"check-postupgrade-state.sh"}
+	line = []string{"check-postupgrade-state.sh", os.Getenv("SHORT_SHA")}
 	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
 		t.Fatalf("fail to check postupgrade state: %v", err)
 	}
@@ -77,11 +77,18 @@ func TestSingleNodeInstallationAlmaLinux8(t *testing.T) {
 	}
 
 	t.Logf("%s: installing embedded-cluster on node 0", time.Now().Format(time.RFC3339))
-	line = []string{"single-node-install.sh", "cli"}
+	line = []string{"single-node-install.sh", "ui"}
 	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
 		t.Fatalf("fail to install embedded-cluster on node %s: %v", tc.Nodes[0], err)
 	}
 
+	if err := setupPlaywright(t, tc); err != nil {
+		t.Fatalf("fail to setup playwright: %v", err)
+	}
+	if _, _, err := runPlaywrightTest(t, tc, "deploy-app"); err != nil {
+		t.Fatalf("fail to run playwright test deploy-app: %v", err)
+	}
+
 	t.Logf("%s: checking installation state", time.Now().Format(time.RFC3339))
 	line = []string{"check-installation-state.sh", os.Getenv("SHORT_SHA")}
 	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
@@ -97,7 +104,7 @@ func TestSingleNodeInstallationAlmaLinux8(t *testing.T) {
 	}
 
 	t.Logf("%s: checking installation state after upgrade", time.Now().Format(time.RFC3339))
-	line = []string{"check-postupgrade-state.sh"}
+	line = []string{"check-postupgrade-state.sh", os.Getenv("SHORT_SHA")}
 	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
 		t.Fatalf("fail to check postupgrade state: %v", err)
 	}
@@ -130,11 +137,18 @@ func TestSingleNodeInstallationDebian12(t *testing.T) {
 	}
 
 	t.Logf("%s: installing embedded-cluster on node 0", time.Now().Format(time.RFC3339))
-	line := []string{"single-node-install.sh", "cli"}
+	line := []string{"single-node-install.sh", "ui"}
 	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
 		t.Fatalf("fail to install embedded-cluster on node %s: %v", tc.Nodes[0], err)
 	}
 
+	if err := setupPlaywright(t, tc); err != nil {
+		t.Fatalf("fail to setup playwright: %v", err)
+	}
+	if _, _, err := runPlaywrightTest(t, tc, "deploy-app"); err != nil {
+		t.Fatalf("fail to run playwright test deploy-app: %v", err)
+	}
+
 	t.Logf("%s: checking installation state", time.Now().Format(time.RFC3339))
 	line = []string{"check-installation-state.sh", os.Getenv("SHORT_SHA")}
 	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
@@ -150,7 +164,7 @@ func TestSingleNodeInstallationDebian12(t *testing.T) {
 	}
 
 	t.Logf("%s: checking installation state after upgrade", time.Now().Format(time.RFC3339))
-	line = []string{"check-postupgrade-state.sh"}
+	line = []string{"check-postupgrade-state.sh", os.Getenv("SHORT_SHA")}
 	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
 		t.Fatalf("fail to check postupgrade state: %v", err)
 	}
@@ -183,11 +197,18 @@ func TestSingleNodeInstallationDebian11(t *testing.T) {
 	}
 
 	t.Logf("%s: installing embedded-cluster on node 0", time.Now().Format(time.RFC3339))
-	line := []string{"single-node-install.sh", "cli"}
+	line := []string{"single-node-install.sh", "ui"}
 	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
 		t.Fatalf("fail to install embedded-cluster on node %s: %v", tc.Nodes[0], err)
 	}
 
+	if err := setupPlaywright(t, tc); err != nil {
+		t.Fatalf("fail to setup playwright: %v", err)
+	}
+	if _, _, err := runPlaywrightTest(t, tc, "deploy-app"); err != nil {
+		t.Fatalf("fail to run playwright test deploy-app: %v", err)
+	}
+
 	t.Logf("%s: checking installation state", time.Now().Format(time.RFC3339))
 	line = []string{"check-installation-state.sh", os.Getenv("SHORT_SHA")}
 	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
@@ -203,7 +224,7 @@ func TestSingleNodeInstallationDebian11(t *testing.T) {
 	}
 
 	t.Logf("%s: checking installation state after upgrade", time.Now().Format(time.RFC3339))
-	line = []string{"check-postupgrade-state.sh"}
+	line = []string{"check-postupgrade-state.sh", os.Getenv("SHORT_SHA")}
 	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
 		t.Fatalf("fail to check postupgrade state: %v", err)
 	}
@@ -232,11 +253,18 @@ func TestSingleNodeInstallationCentos9Stream(t *testing.T) {
 	}
 
 	t.Logf("%s: installing embedded-cluster on node 0", time.Now().Format(time.RFC3339))
-	line = []string{"single-node-install.sh", "cli"}
+	line = []string{"single-node-install.sh", "ui"}
 	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
 		t.Fatalf("fail to install embedded-cluster on node %s: %v", tc.Nodes[0], err)
 	}
 
+	if err := setupPlaywright(t, tc); err != nil {
+		t.Fatalf("fail to setup playwright: %v", err)
+	}
+	if _, _, err := runPlaywrightTest(t, tc, "deploy-app"); err != nil {
+		t.Fatalf("fail to run playwright test deploy-app: %v", err)
+	}
+
 	t.Logf("%s: checking installation state", time.Now().Format(time.RFC3339))
 	line = []string{"check-installation-state.sh", os.Getenv("SHORT_SHA")}
 	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
@@ -252,7 +280,7 @@ func TestSingleNodeInstallationCentos9Stream(t *testing.T) {
 	}
 
 	t.Logf("%s: checking installation state after upgrade", time.Now().Format(time.RFC3339))
-	line = []string{"check-postupgrade-state.sh"}
+	line = []string{"check-postupgrade-state.sh", os.Getenv("SHORT_SHA")}
 	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
 		t.Fatalf("fail to check postupgrade state: %v", err)
 	}
@@ -421,11 +449,18 @@ func TestInstallFromReplicatedApp(t *testing.T) {
 	}
 
 	t.Logf("%s: installing embedded-cluster on node 0", time.Now().Format(time.RFC3339))
-	line = []string{"single-node-install.sh", "cli"}
+	line = []string{"single-node-install.sh", "ui"}
 	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
 		t.Fatalf("fail to install embedded-cluster on node %s: %v", tc.Nodes[0], err)
 	}
 
+	if err := setupPlaywright(t, tc); err != nil {
+		t.Fatalf("fail to setup playwright: %v", err)
+	}
+	if _, _, err := runPlaywrightTest(t, tc, "deploy-app"); err != nil {
+		t.Fatalf("fail to run playwright test deploy-app: %v", err)
+	}
+
 	t.Logf("%s: checking installation state", time.Now().Format(time.RFC3339))
 	line = []string{"check-installation-state.sh", os.Getenv("SHORT_SHA")}
 	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
@@ -441,7 +476,7 @@ func TestInstallFromReplicatedApp(t *testing.T) {
 	}
 
 	t.Logf("%s: checking installation state after upgrade", time.Now().Format(time.RFC3339))
-	line = []string{"check-postupgrade-state.sh"}
+	line = []string{"check-postupgrade-state.sh", os.Getenv("SHORT_SHA")}
 	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
 		t.Fatalf("fail to check postupgrade state: %v", err)
 	}
@@ -464,11 +499,18 @@ func TestResetAndReinstall(t *testing.T) {
 	defer cleanupCluster(t, tc)
 
 	t.Logf("%s: installing embedded-cluster on node 0", time.Now().Format(time.RFC3339))
-	line := []string{"single-node-install.sh", "cli"}
+	line := []string{"single-node-install.sh", "ui"}
 	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
 		t.Fatalf("fail to install embedded-cluster on node %s: %v", tc.Nodes[0], err)
 	}
 
+	if err := setupPlaywright(t, tc); err != nil {
+		t.Fatalf("fail to setup playwright: %v", err)
+	}
+	if _, _, err := runPlaywrightTest(t, tc, "deploy-app"); err != nil {
+		t.Fatalf("fail to run playwright test deploy-app: %v", err)
+	}
+
 	t.Logf("%s: checking installation state", time.Now().Format(time.RFC3339))
 	line = []string{"check-installation-state.sh", os.Getenv("SHORT_SHA")}
 	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
@@ -482,11 +524,15 @@ func TestResetAndReinstall(t *testing.T) {
 	}
 
 	t.Logf("%s: installing embedded-cluster on node 0 after reset", time.Now().Format(time.RFC3339))
-	line = []string{"single-node-install.sh", "cli"}
+	line = []string{"single-node-install.sh", "ui"}
 	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
 		t.Fatalf("fail to install embedded-cluster on node %s: %v", tc.Nodes[0], err)
 	}
 
+	if _, _, err := runPlaywrightTest(t, tc, "deploy-app"); err != nil {
+		t.Fatalf("fail to run playwright test deploy-app: %v", err)
+	}
+
 	t.Logf("%s: checking installation state after reinstall", time.Now().Format(time.RFC3339))
 	line = []string{"check-installation-state.sh", os.Getenv("SHORT_SHA")}
 	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
@@ -609,7 +655,7 @@ func TestOldVersionUpgrade(t *testing.T) {
 	}
 
 	t.Logf("%s: checking installation state after upgrade", time.Now().Format(time.RFC3339))
-	line = []string{"check-postupgrade-state.sh"}
+	line = []string{"check-postupgrade-state.sh", os.Getenv("SHORT_SHA")}
 	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
 		t.Fatalf("fail to check postupgrade state: %v", err)
 	}
@@ -715,7 +761,7 @@ func TestSingleNodeAirgapUpgrade(t *testing.T) {
 	}
 
 	t.Logf("%s: checking installation state after upgrade", time.Now().Format(time.RFC3339))
-	line = []string{"check-postupgrade-state.sh"}
+	line = []string{"check-postupgrade-state.sh", os.Getenv("SHORT_SHA")}
 	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
 		t.Fatalf("fail to check postupgrade state: %v", err)
 	}
@@ -823,7 +869,7 @@ func TestSingleNodeAirgapUpgradeCustomCIDR(t *testing.T) {
 	}
 
 	t.Logf("%s: checking installation state after upgrade", time.Now().Format(time.RFC3339))
-	line = []string{"check-postupgrade-state.sh"}
+	line = []string{"check-postupgrade-state.sh", os.Getenv("SHORT_SHA")}
 	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
 		t.Fatalf("fail to check postupgrade state: %v", err)
 	}
@@ -999,7 +1045,7 @@ func TestMultiNodeAirgapUpgradeSameK0s(t *testing.T) {
 	}
 
 	t.Logf("%s: checking installation state after upgrade", time.Now().Format(time.RFC3339))
-	line = []string{"check-postupgrade-state.sh"}
+	line = []string{"check-postupgrade-state.sh", os.Getenv("SHORT_SHA")}
 	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
 		t.Fatalf("fail to check postupgrade state: %v", err)
 	}
@@ -1159,7 +1205,7 @@ func TestMultiNodeAirgapUpgrade(t *testing.T) {
 	}
 
 	t.Logf("%s: checking installation state after upgrade", time.Now().Format(time.RFC3339))
-	line = []string{"check-postupgrade-state.sh"}
+	line = []string{"check-postupgrade-state.sh", os.Getenv("SHORT_SHA")}
 	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
 		t.Fatalf("fail to check postupgrade state: %v", err)
 	}
@@ -1462,11 +1508,18 @@ func TestInstallSnapshotFromReplicatedApp(t *testing.T) {
 	}
 
 	t.Logf("%s: installing embedded-cluster on node 0", time.Now().Format(time.RFC3339))
-	line = []string{"single-node-install.sh", "cli"}
+	line = []string{"single-node-install.sh", "ui"}
 	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
 		t.Fatalf("fail to install embedded-cluster on node %s: %v", tc.Nodes[0], err)
 	}
 
+	if err := setupPlaywright(t, tc); err != nil {
+		t.Fatalf("fail to setup playwright: %v", err)
+	}
+	if _, _, err := runPlaywrightTest(t, tc, "deploy-app"); err != nil {
+		t.Fatalf("fail to run playwright test deploy-app: %v", err)
+	}
+
 	t.Logf("%s: checking installation state", time.Now().Format(time.RFC3339))
 	line = []string{"check-installation-state.sh", os.Getenv("SHORT_SHA")}
 	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {

From 995461313465d77874a1475a12ba2dd41d65ddae Mon Sep 17 00:00:00 2001
From: Ethan Mosbaugh <ethan@replicated.com>
Date: Wed, 10 Jul 2024 06:29:37 -0700
Subject: [PATCH 58/96] upgrade test

---
 e2e/scripts/install-playwright.sh | 31 +++++++++++++++++++++----------
 1 file changed, 21 insertions(+), 10 deletions(-)

diff --git a/e2e/scripts/install-playwright.sh b/e2e/scripts/install-playwright.sh
index 5005b48f7..c063e0ea3 100755
--- a/e2e/scripts/install-playwright.sh
+++ b/e2e/scripts/install-playwright.sh
@@ -1,21 +1,32 @@
 #!/usr/bin/env bash
 set -euox pipefail
 
-main() {
-    apt-get update -y
-    apt-get install -y \
-        ca-certificates \
-        curl \
-        gnupg
+install_apt() {
+    curl -fsSL "https://deb.nodesource.com/setup_${NODE_MAJOR}.x" -o nodesource_setup.sh
+    bash nodesource_setup.sh
+    apt-get install -y nodejs
+}
 
-    curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg
-    NODE_MAJOR=20
-    echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list
-    apt-get update && apt-get install nodejs -y
+install_yum() {
+    curl -fsSL "https://rpm.nodesource.com/setup_${NODE_MAJOR}.x" -o nodesource_setup.sh
+    bash nodesource_setup.sh
+    yum install -y nodejs
+}
+
+main() {
+    if command -v apt-get &> /dev/null; then
+        install_apt
+    elif command -v yum &> /dev/null; then
+        install_yum
+    else
+        echo "Unsupported package manager"
+        exit 1
+    fi
 
     cd /automation/playwright
     npm ci
     npx playwright install --with-deps
 }
 
+export NODE_MAJOR=20
 main "$@"

From 9800053d2d2c7a38a254c54011ed418b05357f47 Mon Sep 17 00:00:00 2001
From: Ethan Mosbaugh <ethan@replicated.com>
Date: Wed, 10 Jul 2024 11:22:33 -0700
Subject: [PATCH 59/96] f

---
 e2e/cluster/cluster.go            | 13 +++++++------
 e2e/scripts/install-playwright.sh |  8 --------
 2 files changed, 7 insertions(+), 14 deletions(-)

diff --git a/e2e/cluster/cluster.go b/e2e/cluster/cluster.go
index a25d1b6b0..718cd33fb 100644
--- a/e2e/cluster/cluster.go
+++ b/e2e/cluster/cluster.go
@@ -221,13 +221,14 @@ func NewTestCluster(in *Input) *Output {
 			CreateRegularUser(in, node)
 		}
 	}
+	// We create a proxy node for all installations to run playwright tests.
+	out.Proxy = CreateProxy(in)
+	CopyDirsToNode(in, out.Proxy)
+	if in.CreateRegularUser {
+		CreateRegularUser(in, out.Proxy)
+	}
+	NodeHasInternet(in, out.Proxy)
 	if in.WithProxy {
-		out.Proxy = CreateProxy(in)
-		CopyDirsToNode(in, out.Proxy)
-		if in.CreateRegularUser {
-			CreateRegularUser(in, out.Proxy)
-		}
-		NodeHasInternet(in, out.Proxy)
 		ConfigureProxy(in)
 	}
 	return out
diff --git a/e2e/scripts/install-playwright.sh b/e2e/scripts/install-playwright.sh
index c063e0ea3..619eb9d4a 100755
--- a/e2e/scripts/install-playwright.sh
+++ b/e2e/scripts/install-playwright.sh
@@ -7,17 +7,9 @@ install_apt() {
     apt-get install -y nodejs
 }
 
-install_yum() {
-    curl -fsSL "https://rpm.nodesource.com/setup_${NODE_MAJOR}.x" -o nodesource_setup.sh
-    bash nodesource_setup.sh
-    yum install -y nodejs
-}
-
 main() {
     if command -v apt-get &> /dev/null; then
         install_apt
-    elif command -v yum &> /dev/null; then
-        install_yum
     else
         echo "Unsupported package manager"
         exit 1

From 542b1fdcef6ee3bb0401427b5df1f007a1b16399 Mon Sep 17 00:00:00 2001
From: Ethan Mosbaugh <ethan@replicated.com>
Date: Wed, 10 Jul 2024 11:39:10 -0700
Subject: [PATCH 60/96] f

---
 e2e/cluster/cluster.go | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/e2e/cluster/cluster.go b/e2e/cluster/cluster.go
index 718cd33fb..71e49ece2 100644
--- a/e2e/cluster/cluster.go
+++ b/e2e/cluster/cluster.go
@@ -210,7 +210,10 @@ func NewTestCluster(in *Input) *Output {
 	}
 	out.T.Cleanup(out.Destroy)
 
-	PullImage(in)
+	PullImage(in, in.Image)
+	if ProxyImage != in.Image {
+		PullImage(in, ProxyImage)
+	}
 	CreateProfile(in)
 	CreateNetworks(in)
 	out.Nodes = CreateNodes(in)
@@ -234,6 +237,7 @@ func NewTestCluster(in *Input) *Output {
 	return out
 }
 
+const ProxyImage = "debian/12"
 const HTTPProxy = "http://10.0.0.254:3128"
 const NOProxy = "10.0.0.0/8"
 
@@ -256,7 +260,7 @@ func CreateProxy(in *Input) string {
 		Type: api.InstanceTypeContainer,
 		Source: api.InstanceSource{
 			Type:  "image",
-			Alias: "debian/12",
+			Alias: ProxyImage,
 		},
 		InstancePut: api.InstancePut{
 			Profiles:     []string{profile},
@@ -740,7 +744,7 @@ func CreateProfile(in *Input) {
 }
 
 // PullImage pull the image used for the nodes.
-func PullImage(in *Input) {
+func PullImage(in *Input, image string) {
 	client, err := lxd.ConnectLXDUnix(lxdSocket, nil)
 	if err != nil {
 		in.T.Fatalf("Failed to connect to LXD: %v", err)
@@ -750,15 +754,15 @@ func PullImage(in *Input) {
 		"https://images.lxd.canonical.com",
 		"https://cloud-images.ubuntu.com/minimal/releases",
 	} {
-		in.T.Logf("Pulling %q image from %s", in.Image, server)
+		in.T.Logf("Pulling %q image from %s", image, server)
 		remote, err := lxd.ConnectSimpleStreams(server, nil)
 		if err != nil {
 			in.T.Fatalf("Failed to connect to image server: %v", err)
 		}
 
-		alias, _, err := remote.GetImageAlias(in.Image)
+		alias, _, err := remote.GetImageAlias(image)
 		if err != nil {
-			in.T.Logf("Failed to get image alias %s on %s: %v", in.Image, server, err)
+			in.T.Logf("Failed to get image alias %s on %s: %v", image, server, err)
 			continue
 		}
 
@@ -780,5 +784,5 @@ func PullImage(in *Input) {
 		in.T.Logf("Failed to wait for image copy: %v", err)
 	}
 
-	in.T.Fatalf("Failed to pull image %s (tried in all servers)", in.Image)
+	in.T.Fatalf("Failed to pull image %s (tried in all servers)", image)
 }

From abf6e2a98aa06b9e89346ec01184759c9d76e425 Mon Sep 17 00:00:00 2001
From: Ethan Mosbaugh <ethan@replicated.com>
Date: Wed, 10 Jul 2024 12:13:51 -0700
Subject: [PATCH 61/96] f

---
 e2e/scripts/install-playwright.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/e2e/scripts/install-playwright.sh b/e2e/scripts/install-playwright.sh
index 619eb9d4a..130d78d9e 100755
--- a/e2e/scripts/install-playwright.sh
+++ b/e2e/scripts/install-playwright.sh
@@ -2,6 +2,7 @@
 set -euox pipefail
 
 install_apt() {
+    apt-get update && apt-get install -y curl
     curl -fsSL "https://deb.nodesource.com/setup_${NODE_MAJOR}.x" -o nodesource_setup.sh
     bash nodesource_setup.sh
     apt-get install -y nodejs

From 02ad14ceb60d138eaa4abd1f531794f5d0c3a399 Mon Sep 17 00:00:00 2001
From: Ethan Mosbaugh <ethan@replicated.com>
Date: Wed, 10 Jul 2024 13:02:43 -0700
Subject: [PATCH 62/96] f

---
 e2e/cluster/cluster.go | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/e2e/cluster/cluster.go b/e2e/cluster/cluster.go
index 71e49ece2..a8d9e5f75 100644
--- a/e2e/cluster/cluster.go
+++ b/e2e/cluster/cluster.go
@@ -672,12 +672,10 @@ func CreateNetworks(in *Input) {
 				"ipv4.dhcp.ranges": fmt.Sprintf("%[1]s.2-%[1]s.254", in.network),
 				"ipv4.nat":         "true",
 				"ipv4.ovn.ranges":  fmt.Sprintf("%[1]s.100-%[1]s.253", in.network),
+				"ipv4.routes":      "10.0.0.0/24",
 			},
 		},
 	}
-	if in.WithProxy {
-		request.NetworkPut.Config["ipv4.routes"] = "10.0.0.0/24"
-	}
 	if err := client.CreateNetwork(request); err != nil {
 		in.T.Fatalf("Failed to create external network: %v", err)
 	}

From 69a7648e1ac27154ce33e4272142cce80f1e2dad Mon Sep 17 00:00:00 2001
From: Ethan Mosbaugh <ethan@replicated.com>
Date: Wed, 10 Jul 2024 13:20:58 -0700
Subject: [PATCH 63/96] f

---
 e2e/cluster/cluster.go | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/e2e/cluster/cluster.go b/e2e/cluster/cluster.go
index a8d9e5f75..f13c30103 100644
--- a/e2e/cluster/cluster.go
+++ b/e2e/cluster/cluster.go
@@ -231,6 +231,7 @@ func NewTestCluster(in *Input) *Output {
 		CreateRegularUser(in, out.Proxy)
 	}
 	NodeHasInternet(in, out.Proxy)
+	ConfigureProxyNode(in)
 	if in.WithProxy {
 		ConfigureProxy(in)
 	}
@@ -304,14 +305,14 @@ func CreateProxy(in *Input) string {
 	return name
 }
 
-// ConfigureProxy installs squid and iptables on the target node. Configures the needed
+// ConfigureProxyNode installs squid and iptables on the target node. Configures the needed
 // ip addresses and sets up iptables to allow nat for requests coming out on eth0 using
-// port 53(UDP). Configures squid to accept requests coming from 10.0.0.0/24 network.
-// Proxy will be listening on http://10.0.0.254:3128.
-func ConfigureProxy(in *Input) {
+// port 53(UDP).
+func ConfigureProxyNode(in *Input) {
+	proxyName := fmt.Sprintf("node-%s-proxy", in.id)
+
 	// starts by installing dependencies, setting up the second network interface ip
 	// address and configuring iptables to allow dns requests forwarding (nat).
-	proxyName := fmt.Sprintf("node-%s-proxy", in.id)
 	for _, cmd := range [][]string{
 		{"apt-get", "update", "-y"},
 		{"apt-get", "install", "-y", "iptables", "squid"},
@@ -322,6 +323,14 @@ func ConfigureProxy(in *Input) {
 	} {
 		RunCommandOnNode(in, cmd, proxyName)
 	}
+}
+
+// ConfigureProxy configures squid to accept requests coming from 10.0.0.0/24 network.
+// Proxy will be listening on http://10.0.0.254:3128. It also sets the default route
+// on all other nodes to point to the proxy to ensure no internet will work on them
+// other than dns and http requests using the proxy.
+func ConfigureProxy(in *Input) {
+	proxyName := fmt.Sprintf("node-%s-proxy", in.id)
 
 	// create a simple squid configuration that allows for localnet access. upload it
 	// to the proxy in the right location. restart squid to apply the configuration.

From 8805150e8a3ad4947898ce2a836303c45eb953a2 Mon Sep 17 00:00:00 2001
From: Ethan Mosbaugh <ethan@replicated.com>
Date: Wed, 10 Jul 2024 13:57:04 -0700
Subject: [PATCH 64/96] f

---
 e2e/install_test.go       | 42 +++++++++++++++++++++++++++------------
 e2e/restore_test.go       | 11 ++++++++--
 e2e/scripts/playwright.sh | 11 +++++-----
 3 files changed, 43 insertions(+), 21 deletions(-)

diff --git a/e2e/install_test.go b/e2e/install_test.go
index c6886e076..80cc89062 100644
--- a/e2e/install_test.go
+++ b/e2e/install_test.go
@@ -666,7 +666,7 @@ func TestOldVersionUpgrade(t *testing.T) {
 func TestSingleNodeAirgapUpgrade(t *testing.T) {
 	t.Parallel()
 
-	RequireEnvVars(t, []string{"SHORT_SHA"})
+	RequireEnvVars(t, []string{"SHORT_SHA", "AIRGAP_LICENSE_ID"})
 
 	t.Logf("%s: downloading airgap files", time.Now().Format(time.RFC3339))
 	airgapInstallBundlePath := "/tmp/airgap-install-bundle.tar.gz"
@@ -756,8 +756,12 @@ func TestSingleNodeAirgapUpgrade(t *testing.T) {
 		t.Fatalf("fail to remove airgap bundle on node %s: %v", tc.Nodes[0], err)
 	}
 
-	if _, _, err := runPlaywrightTest(t, tc, "deploy-airgap-upgrade"); err != nil {
-		t.Fatalf("fail to run playwright test deploy-airgap-upgrade: %v", err)
+	appUpgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA"))
+	testArgs := []string{appUpgradeVersion}
+
+	t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339))
+	if _, _, err := runPlaywrightTest(t, tc, "deploy-upgrade", testArgs...); err != nil {
+		t.Fatalf("fail to run playwright test deploy-app: %v", err)
 	}
 
 	t.Logf("%s: checking installation state after upgrade", time.Now().Format(time.RFC3339))
@@ -772,7 +776,7 @@ func TestSingleNodeAirgapUpgrade(t *testing.T) {
 func TestSingleNodeAirgapUpgradeCustomCIDR(t *testing.T) {
 	t.Parallel()
 
-	RequireEnvVars(t, []string{"SHORT_SHA"})
+	RequireEnvVars(t, []string{"SHORT_SHA", "AIRGAP_LICENSE_ID"})
 
 	t.Logf("%s: downloading airgap files", time.Now().Format(time.RFC3339))
 	airgapInstallBundlePath := "/tmp/airgap-install-bundle.tar.gz"
@@ -864,8 +868,12 @@ func TestSingleNodeAirgapUpgradeCustomCIDR(t *testing.T) {
 		t.Fatalf("fail to remove airgap bundle on node %s: %v", tc.Nodes[0], err)
 	}
 
-	if _, _, err := runPlaywrightTest(t, tc, "deploy-airgap-upgrade"); err != nil {
-		t.Fatalf("fail to run playwright test deploy-airgap-upgrade: %v", err)
+	appUpgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA"))
+	testArgs := []string{appUpgradeVersion}
+
+	t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339))
+	if _, _, err := runPlaywrightTest(t, tc, "deploy-upgrade", testArgs...); err != nil {
+		t.Fatalf("fail to run playwright test deploy-app: %v", err)
 	}
 
 	t.Logf("%s: checking installation state after upgrade", time.Now().Format(time.RFC3339))
@@ -888,7 +896,7 @@ func TestSingleNodeAirgapUpgradeCustomCIDR(t *testing.T) {
 func TestMultiNodeAirgapUpgradeSameK0s(t *testing.T) {
 	t.Parallel()
 
-	RequireEnvVars(t, []string{"SHORT_SHA"})
+	RequireEnvVars(t, []string{"SHORT_SHA", "AIRGAP_LICENSE_ID"})
 
 	t.Logf("%s: downloading airgap files", time.Now().Format(time.RFC3339))
 	airgapInstallBundlePath := "/tmp/airgap-install-bundle.tar.gz"
@@ -1040,8 +1048,12 @@ func TestMultiNodeAirgapUpgradeSameK0s(t *testing.T) {
 		t.Fatalf("fail to remove embedded-cluster-upgrade binary on node %s: %v", tc.Nodes[0], err)
 	}
 
-	if _, _, err := runPlaywrightTest(t, tc, "deploy-airgap-upgrade", "true"); err != nil {
-		t.Fatalf("fail to run playwright test deploy-airgap-upgrade: %v", err)
+	appUpgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA"))
+	testArgs := []string{appUpgradeVersion, "true"} // true to indicate that this
+
+	t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339))
+	if _, _, err := runPlaywrightTest(t, tc, "deploy-upgrade", testArgs...); err != nil {
+		t.Fatalf("fail to run playwright test deploy-app: %v", err)
 	}
 
 	t.Logf("%s: checking installation state after upgrade", time.Now().Format(time.RFC3339))
@@ -1056,7 +1068,7 @@ func TestMultiNodeAirgapUpgradeSameK0s(t *testing.T) {
 func TestMultiNodeAirgapUpgrade(t *testing.T) {
 	t.Parallel()
 
-	RequireEnvVars(t, []string{"SHORT_SHA"})
+	RequireEnvVars(t, []string{"SHORT_SHA", "AIRGAP_LICENSE_ID"})
 
 	t.Logf("%s: downloading airgap files", time.Now().Format(time.RFC3339))
 	airgapInstallBundlePath := "/tmp/airgap-install-bundle.tar.gz"
@@ -1200,8 +1212,12 @@ func TestMultiNodeAirgapUpgrade(t *testing.T) {
 		t.Fatalf("fail to remove embedded-cluster-upgrade binary on node %s: %v", tc.Nodes[0], err)
 	}
 
-	if _, _, err := runPlaywrightTest(t, tc, "deploy-airgap-upgrade"); err != nil {
-		t.Fatalf("fail to run playwright test deploy-airgap-upgrade: %v", err)
+	appUpgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA"))
+	testArgs := []string{appUpgradeVersion}
+
+	t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339))
+	if _, _, err := runPlaywrightTest(t, tc, "deploy-upgrade", testArgs...); err != nil {
+		t.Fatalf("fail to run playwright test deploy-app: %v", err)
 	}
 
 	t.Logf("%s: checking installation state after upgrade", time.Now().Format(time.RFC3339))
@@ -1492,7 +1508,7 @@ func TestMultiNodeAirgapHAInstallation(t *testing.T) {
 func TestInstallSnapshotFromReplicatedApp(t *testing.T) {
 	t.Parallel()
 
-	RequireEnvVars(t, []string{"SHORT_SHA"})
+	RequireEnvVars(t, []string{"SHORT_SHA", "SNAPSHOT_LICENSE_ID"})
 
 	tc := cluster.NewTestCluster(&cluster.Input{
 		T:     t,
diff --git a/e2e/restore_test.go b/e2e/restore_test.go
index a1a87092f..66b5f49f2 100644
--- a/e2e/restore_test.go
+++ b/e2e/restore_test.go
@@ -235,6 +235,9 @@ func TestSingleNodeResumeDisasterRecovery(t *testing.T) {
 
 func TestSingleNodeAirgapDisasterRecovery(t *testing.T) {
 	t.Parallel()
+
+	RequireEnvVars(t, []string{"SHORT_SHA", "AIRGAP_SNAPSHOT_LICENSE_ID"})
+
 	requiredEnvVars := []string{
 		"DR_AWS_S3_ENDPOINT",
 		"DR_AWS_S3_REGION",
@@ -378,8 +381,12 @@ func TestSingleNodeAirgapDisasterRecovery(t *testing.T) {
 		t.Fatalf("fail to remove airgap bundle on node %s: %v", tc.Nodes[0], err)
 	}
 
-	if _, _, err := runPlaywrightTest(t, tc, "deploy-airgap-upgrade"); err != nil {
-		t.Fatalf("fail to run playwright test deploy-airgap-upgrade: %v", err)
+	appUpgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA"))
+	testArgs = []string{appUpgradeVersion}
+
+	t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339))
+	if _, _, err := runPlaywrightTest(t, tc, "deploy-upgrade", testArgs...); err != nil {
+		t.Fatalf("fail to run playwright test deploy-app: %v", err)
 	}
 
 	t.Logf("%s: checking installation state after upgrade", time.Now().Format(time.RFC3339))
diff --git a/e2e/scripts/playwright.sh b/e2e/scripts/playwright.sh
index 03ecbd968..780c48a65 100755
--- a/e2e/scripts/playwright.sh
+++ b/e2e/scripts/playwright.sh
@@ -17,15 +17,14 @@ main() {
     export DR_AWS_SECRET_ACCESS_KEY="$7"
   elif [ "$test_name" == "deploy-upgrade" ]; then
     export APP_UPGRADE_VERSION="$2"
-  fi
 
-  # if we have a second argument and the first points to the
-  # deploy-airgap-upgrade script we set the env variable to
-  # make the test skip the "Cluster update in progress" check.
-  if [ $# -ge 2 ] && [ "$test_name" == "deploy-airgap-upgrade" ]; then
-    export SKIP_CLUSTER_UPGRADING_CHECK="$2"
+    # make the test skip the "Cluster update in progress" check.
+    if [ $# -ge 3 ] && [ "$3" == "true" ]; then
+      export SKIP_CLUSTER_UPGRADING_CHECK="true" # TODO
+    fi
   fi
 
+
   export BASE_URL="http://10.0.0.2:30001"
   cd /automation/playwright
   npx playwright test "$test_name"

From 6776b1c1ea57a4908c2fc1450ac574f091561e52 Mon Sep 17 00:00:00 2001
From: Ethan Mosbaugh <ethan@replicated.com>
Date: Wed, 10 Jul 2024 14:51:01 -0700
Subject: [PATCH 65/96] f

---
 e2e/install_test.go                              |  4 ++--
 e2e/kots-release-install/preflight.yaml          | 12 ------------
 e2e/kots-release-upgrade/preflight.yaml          | 12 ------------
 e2e/local-artifact-mirror_test.go                |  6 +++++-
 e2e/materialize_test.go                          |  2 +-
 e2e/playwright/tests/deploy-upgrade/test.spec.ts |  4 ++--
 e2e/scripts/playwright.sh                        |  5 -----
 e2e/sudo_test.go                                 |  2 +-
 e2e/support-bundle_test.go                       |  2 +-
 e2e/unsupported-overrides_test.go                |  2 +-
 e2e/version_test.go                              |  2 +-
 11 files changed, 14 insertions(+), 39 deletions(-)

diff --git a/e2e/install_test.go b/e2e/install_test.go
index 80cc89062..244b7a243 100644
--- a/e2e/install_test.go
+++ b/e2e/install_test.go
@@ -301,7 +301,7 @@ func TestHostPreflight(t *testing.T) {
 		EmbeddedClusterPath:               "../output/bin/embedded-cluster",
 		EmbeddedClusterReleaseBuilderPath: "../output/bin/embedded-cluster-release-builder",
 	})
-	defer tc.Destroy()
+	defer cleanupCluster(t, tc)
 
 	t.Logf("%s: installing test dependencies on node 0", time.Now().Format(time.RFC3339))
 	commands := [][]string{
@@ -1049,7 +1049,7 @@ func TestMultiNodeAirgapUpgradeSameK0s(t *testing.T) {
 	}
 
 	appUpgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA"))
-	testArgs := []string{appUpgradeVersion, "true"} // true to indicate that this
+	testArgs := []string{appUpgradeVersion}
 
 	t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339))
 	if _, _, err := runPlaywrightTest(t, tc, "deploy-upgrade", testArgs...); err != nil {
diff --git a/e2e/kots-release-install/preflight.yaml b/e2e/kots-release-install/preflight.yaml
index 1c1fa9376..9eb4ce865 100644
--- a/e2e/kots-release-install/preflight.yaml
+++ b/e2e/kots-release-install/preflight.yaml
@@ -4,18 +4,6 @@ metadata:
   name: preflight-checks
 spec:
   analyzers:
-    - clusterVersion:
-        outcomes:
-          - fail:
-              when: "< 1.28.0"
-              message: The application requires at Kubernetes 1.28.0 or later, and recommends 1.29.0.
-              uri: https://www.kubernetes.io
-          - warn:
-              when: "< 1.29.0"
-              message: Your cluster meets the minimum version of Kubernetes, but we recommend you update to 1.29.0 or later.
-              uri: https://kubernetes.io
-          - pass:
-              message: Your cluster meets the recommended and required versions of Kubernetes.
     - customResourceDefinition:
         customResourceDefinitionName: installations.embeddedcluster.replicated.com
         checkName: Embedded Cluster Installation CRD exists
diff --git a/e2e/kots-release-upgrade/preflight.yaml b/e2e/kots-release-upgrade/preflight.yaml
index 1c1fa9376..9eb4ce865 100644
--- a/e2e/kots-release-upgrade/preflight.yaml
+++ b/e2e/kots-release-upgrade/preflight.yaml
@@ -4,18 +4,6 @@ metadata:
   name: preflight-checks
 spec:
   analyzers:
-    - clusterVersion:
-        outcomes:
-          - fail:
-              when: "< 1.28.0"
-              message: The application requires at Kubernetes 1.28.0 or later, and recommends 1.29.0.
-              uri: https://www.kubernetes.io
-          - warn:
-              when: "< 1.29.0"
-              message: Your cluster meets the minimum version of Kubernetes, but we recommend you update to 1.29.0 or later.
-              uri: https://kubernetes.io
-          - pass:
-              message: Your cluster meets the recommended and required versions of Kubernetes.
     - customResourceDefinition:
         customResourceDefinitionName: installations.embeddedcluster.replicated.com
         checkName: Embedded Cluster Installation CRD exists
diff --git a/e2e/local-artifact-mirror_test.go b/e2e/local-artifact-mirror_test.go
index 2458ae322..0e5562773 100644
--- a/e2e/local-artifact-mirror_test.go
+++ b/e2e/local-artifact-mirror_test.go
@@ -10,13 +10,17 @@ import (
 
 func TestLocalArtifactMirror(t *testing.T) {
 	t.Parallel()
+
+	RequireEnvVars(t, []string{"SHORT_SHA"})
+
 	tc := cluster.NewTestCluster(&cluster.Input{
 		T:                   t,
 		Nodes:               1,
 		Image:               "debian/12",
+		LicensePath:         "license.yaml",
 		EmbeddedClusterPath: "../output/bin/embedded-cluster-original",
 	})
-	defer tc.Destroy()
+	defer cleanupCluster(t, tc)
 
 	t.Logf("%s: installing embedded-cluster on node 0", time.Now().Format(time.RFC3339))
 	line := []string{"default-install.sh"}
diff --git a/e2e/materialize_test.go b/e2e/materialize_test.go
index 249e56413..d4b57f615 100644
--- a/e2e/materialize_test.go
+++ b/e2e/materialize_test.go
@@ -15,7 +15,7 @@ func TestMaterialize(t *testing.T) {
 		Image:               "debian/12",
 		EmbeddedClusterPath: "../output/bin/embedded-cluster-original",
 	})
-	defer tc.Destroy()
+	defer cleanupCluster(t, tc)
 
 	commands := [][]string{
 		{"rm", "-rf", "/var/lib/embedded-cluster/bin/kubectl"},
diff --git a/e2e/playwright/tests/deploy-upgrade/test.spec.ts b/e2e/playwright/tests/deploy-upgrade/test.spec.ts
index 9b12416f1..85cd2f46b 100644
--- a/e2e/playwright/tests/deploy-upgrade/test.spec.ts
+++ b/e2e/playwright/tests/deploy-upgrade/test.spec.ts
@@ -15,8 +15,8 @@ test('deploy upgrade', async ({ page }) => {
   await iframe.getByRole('button', { name: 'Next', exact: true }).click();
   await iframe.getByRole('button', { name: 'Next: Confirm and deploy', exact: true }).click({ timeout: 10 * 1000 });
   await iframe.getByRole('button', { name: 'Deploy', exact: true }).click();
-  await expect(page.locator('.Modal-body').getByText('Cluster update in progress')).toBeVisible();
-  await expect(page.locator('.Modal-body').getByText('Cluster update in progress')).not.toBeVisible({ timeout: 5 * 60 * 1000 });
+  await expect(page.locator('.Modal-body')).toContainText('Cluster update in progress');
+  await expect(page.locator('.Modal-body')).not.toContainText('Cluster update in progress', { timeout: 5 * 60 * 1000 });
   await page.getByRole('link', { name: 'Dashboard', exact: true }).click();
   await expect(page.locator('.VersionCard-content--wrapper')).toContainText(process.env.APP_UPGRADE_VERSION);
   await expect(page.locator('#app')).toContainText('Currently deployed version', { timeout: 2 * 60 * 1000 });
diff --git a/e2e/scripts/playwright.sh b/e2e/scripts/playwright.sh
index 780c48a65..491970624 100755
--- a/e2e/scripts/playwright.sh
+++ b/e2e/scripts/playwright.sh
@@ -17,11 +17,6 @@ main() {
     export DR_AWS_SECRET_ACCESS_KEY="$7"
   elif [ "$test_name" == "deploy-upgrade" ]; then
     export APP_UPGRADE_VERSION="$2"
-
-    # make the test skip the "Cluster update in progress" check.
-    if [ $# -ge 3 ] && [ "$3" == "true" ]; then
-      export SKIP_CLUSTER_UPGRADING_CHECK="true" # TODO
-    fi
   fi
 
 
diff --git a/e2e/sudo_test.go b/e2e/sudo_test.go
index f85384dda..f81116c84 100644
--- a/e2e/sudo_test.go
+++ b/e2e/sudo_test.go
@@ -17,7 +17,7 @@ func TestCommandsRequireSudo(t *testing.T) {
 		Image:               "debian/12",
 		EmbeddedClusterPath: "../output/bin/embedded-cluster",
 	})
-	defer tc.Destroy()
+	defer cleanupCluster(t, tc)
 	t.Logf(`%s: running "embedded-cluster version" as regular user`, time.Now().Format(time.RFC3339))
 	command := []string{"embedded-cluster", "version"}
 	if _, _, err := RunRegularUserCommandOnNode(t, tc, 0, command); err != nil {
diff --git a/e2e/support-bundle_test.go b/e2e/support-bundle_test.go
index e86d79b11..79a5ddb2d 100644
--- a/e2e/support-bundle_test.go
+++ b/e2e/support-bundle_test.go
@@ -16,7 +16,7 @@ func TestCollectSupportBundle(t *testing.T) {
 		LicensePath:         "license.yaml",
 		EmbeddedClusterPath: "../output/bin/embedded-cluster",
 	})
-	defer tc.Destroy()
+	defer cleanupCluster(t, tc)
 
 	t.Logf("%s: installing embedded-cluster on node 0", time.Now().Format(time.RFC3339))
 	line := []string{"single-node-install.sh", "cli"}
diff --git a/e2e/unsupported-overrides_test.go b/e2e/unsupported-overrides_test.go
index 5d36c41de..7ba76b901 100644
--- a/e2e/unsupported-overrides_test.go
+++ b/e2e/unsupported-overrides_test.go
@@ -17,7 +17,7 @@ func TestUnsupportedOverrides(t *testing.T) {
 		EmbeddedClusterPath:               "../output/bin/embedded-cluster",
 		EmbeddedClusterReleaseBuilderPath: "../output/bin/embedded-cluster-release-builder",
 	})
-	defer tc.Destroy()
+	defer cleanupCluster(t, tc)
 	t.Logf("%s: installing dependencies on node 0", time.Now().Format(time.RFC3339))
 	commands := [][]string{
 		{"apt-get", "update", "-y"},
diff --git a/e2e/version_test.go b/e2e/version_test.go
index 0489935d8..7eb57bbe4 100644
--- a/e2e/version_test.go
+++ b/e2e/version_test.go
@@ -20,7 +20,7 @@ func TestVersion(t *testing.T) {
 		Image:               "debian/12",
 		EmbeddedClusterPath: "../output/bin/embedded-cluster",
 	})
-	defer tc.Destroy()
+	defer cleanupCluster(t, tc)
 	t.Logf("%s: validating 'embedded-cluster version' in node 0", time.Now().Format(time.RFC3339))
 	line := []string{"embedded-cluster", "version"}
 	stdout, stderr, err := RunRegularUserCommandOnNode(t, tc, 0, line)

From c8b6f1cb96ea2257b055261b46c951373909c6ea Mon Sep 17 00:00:00 2001
From: Ethan Mosbaugh <ethan@replicated.com>
Date: Wed, 10 Jul 2024 15:15:47 -0700
Subject: [PATCH 66/96] f

---
 .github/actions/e2e/action.yml                |  2 +-
 Makefile                                      |  2 +-
 e2e/install_test.go                           | 30 +++++++++++++------
 .../tests/deploy-upgrade/test.spec.ts         |  2 +-
 e2e/scripts/collect-support-bundle-cluster.sh | 13 ++++++++
 ...ndle.sh => collect-support-bundle-host.sh} |  5 ----
 e2e/support-bundle_test.go                    | 12 ++++++--
 7 files changed, 47 insertions(+), 19 deletions(-)
 create mode 100644 e2e/scripts/collect-support-bundle-cluster.sh
 rename e2e/scripts/{collect-support-bundle.sh => collect-support-bundle-host.sh} (67%)

diff --git a/.github/actions/e2e/action.yml b/.github/actions/e2e/action.yml
index 332e21920..58a248be1 100644
--- a/.github/actions/e2e/action.yml
+++ b/.github/actions/e2e/action.yml
@@ -110,7 +110,7 @@ runs:
     if: ${{ failure() }}
     with:
       name: ${{ inputs.test-name }}-support-bundle-host.tar.gz
-      path: ./e2e/support-bundle-host.tar.gz
+      path: ./e2e/support-bundle-host-*.tar.gz
   - name: Upload Cluster Support Bundle
     uses: actions/upload-artifact@v4
     if: ${{ failure() }}
diff --git a/Makefile b/Makefile
index 3e49f250c..2ee5e3274 100644
--- a/Makefile
+++ b/Makefile
@@ -7,7 +7,7 @@ ADMIN_CONSOLE_CHART_VERSION = 1.110.0
 ADMIN_CONSOLE_IMAGE_OVERRIDE = kotsadm/kotsadm:v2024.7.8-ec8-nightly
 ADMIN_CONSOLE_MIGRATIONS_IMAGE_OVERRIDE = kotsadm/kotsadm-migrations:v2024.7.8-ec8-nightly
 ADMIN_CONSOLE_KURL_PROXY_IMAGE_OVERRIDE = kotsadm/kurl-proxy:v2024.7.8-ec8-nightly
-EMBEDDED_OPERATOR_CHART_VERSION = 0.39.0-build.2
+EMBEDDED_OPERATOR_CHART_VERSION = 0.39.0-build.3
 EMBEDDED_OPERATOR_BINARY_URL_OVERRIDE =
 EMBEDDED_OPERATOR_UTILS_IMAGE = busybox:1.36.1
 EMBEDDED_CLUSTER_OPERATOR_IMAGE_OVERRIDE =
diff --git a/e2e/install_test.go b/e2e/install_test.go
index 244b7a243..dd131b27e 100644
--- a/e2e/install_test.go
+++ b/e2e/install_test.go
@@ -1742,21 +1742,33 @@ func runPlaywrightTest(t *testing.T, tc *cluster.Output, testName string, args .
 }
 
 func generateAndCopySupportBundle(t *testing.T, tc *cluster.Output) {
-	t.Logf("%s: generating support bundle", time.Now().Format(time.RFC3339))
-	line := []string{"collect-support-bundle.sh"}
+	node := tc.Nodes[0]
+	t.Logf("%s: generating cluster support bundle from node %s", time.Now().Format(time.RFC3339), node)
+	line := []string{"collect-support-bundle-cluster.sh"}
 	if stdout, stderr, err := RunCommandOnNode(t, tc, 0, line); err != nil {
 		t.Logf("stdout: %s", stdout)
 		t.Logf("stderr: %s", stderr)
-		t.Errorf("fail to generate support bundle: %v", err)
+		t.Errorf("fail to generate cluster support from node %s bundle: %v", node, err)
 	}
 
-	t.Logf("%s: copying host support bundle to local machine", time.Now().Format(time.RFC3339))
-	if err := cluster.CopyFileFromNode(tc.Nodes[0], "/root/host.tar.gz", "support-bundle-host.tar.gz"); err != nil {
-		t.Errorf("fail to copy host support bundle to local machine: %v", err)
+	t.Logf("%s: copying cluster support bundle from node %s to local machine", time.Now().Format(time.RFC3339), node)
+	if err := cluster.CopyFileFromNode(node, "/root/cluster.tar.gz", "support-bundle-cluster.tar.gz"); err != nil {
+		t.Errorf("fail to copy cluster support bundle from node %s to local machine: %v", node, err)
 	}
-	t.Logf("%s: copying cluster support bundle to local machine", time.Now().Format(time.RFC3339))
-	if err := cluster.CopyFileFromNode(tc.Nodes[0], "/root/cluster.tar.gz", "support-bundle-cluster.tar.gz"); err != nil {
-		t.Errorf("fail to copy cluster support bundle to local machine: %v", err)
+
+	for i, node := range tc.Nodes {
+		t.Logf("%s: generating host support bundle from node %s", time.Now().Format(time.RFC3339), node)
+		line := []string{"collect-support-bundle-host.sh"}
+		if stdout, stderr, err := RunCommandOnNode(t, tc, i, line); err != nil {
+			t.Logf("stdout: %s", stdout)
+			t.Logf("stderr: %s", stderr)
+			t.Errorf("fail to generate support from node %s bundle: %v", node, err)
+		}
+
+		t.Logf("%s: copying host support bundle from node %s to local machine", time.Now().Format(time.RFC3339), node)
+		if err := cluster.CopyFileFromNode(node, "/root/host.tar.gz", fmt.Sprintf("support-bundle-host-%s.tar.gz", node)); err != nil {
+			t.Errorf("fail to copy host support bundle from node %s to local machine: %v", node, err)
+		}
 	}
 }
 
diff --git a/e2e/playwright/tests/deploy-upgrade/test.spec.ts b/e2e/playwright/tests/deploy-upgrade/test.spec.ts
index 85cd2f46b..a94c4d4d8 100644
--- a/e2e/playwright/tests/deploy-upgrade/test.spec.ts
+++ b/e2e/playwright/tests/deploy-upgrade/test.spec.ts
@@ -16,7 +16,7 @@ test('deploy upgrade', async ({ page }) => {
   await iframe.getByRole('button', { name: 'Next: Confirm and deploy', exact: true }).click({ timeout: 10 * 1000 });
   await iframe.getByRole('button', { name: 'Deploy', exact: true }).click();
   await expect(page.locator('.Modal-body')).toContainText('Cluster update in progress');
-  await expect(page.locator('.Modal-body')).not.toContainText('Cluster update in progress', { timeout: 5 * 60 * 1000 });
+  await expect(page.locator('.Modal-body').getByText('Cluster update in progress')).not.toBeVisible({ timeout: 5 * 60 * 1000 });
   await page.getByRole('link', { name: 'Dashboard', exact: true }).click();
   await expect(page.locator('.VersionCard-content--wrapper')).toContainText(process.env.APP_UPGRADE_VERSION);
   await expect(page.locator('#app')).toContainText('Currently deployed version', { timeout: 2 * 60 * 1000 });
diff --git a/e2e/scripts/collect-support-bundle-cluster.sh b/e2e/scripts/collect-support-bundle-cluster.sh
new file mode 100644
index 000000000..ecf05df2f
--- /dev/null
+++ b/e2e/scripts/collect-support-bundle-cluster.sh
@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+set -euox pipefail
+
+main() {
+    if ! kubectl support-bundle --output cluster.tar.gz --interactive=false --load-cluster-specs; then
+        echo "Failed to collect cluster support bundle"
+        return 1
+    fi
+}
+
+export KUBECONFIG=/var/lib/k0s/pki/admin.conf
+export PATH=$PATH:/var/lib/embedded-cluster/bin
+main "$@"
diff --git a/e2e/scripts/collect-support-bundle.sh b/e2e/scripts/collect-support-bundle-host.sh
similarity index 67%
rename from e2e/scripts/collect-support-bundle.sh
rename to e2e/scripts/collect-support-bundle-host.sh
index b26c7403c..28d0de101 100755
--- a/e2e/scripts/collect-support-bundle.sh
+++ b/e2e/scripts/collect-support-bundle-host.sh
@@ -6,11 +6,6 @@ main() {
         echo "Failed to collect local host support bundle"
         return 1
     fi
-
-    if ! kubectl support-bundle --output cluster.tar.gz --interactive=false --load-cluster-specs; then
-        echo "Failed to collect cluster support bundle"
-        return 1
-    fi
 }
 
 export KUBECONFIG=/var/lib/k0s/pki/admin.conf
diff --git a/e2e/support-bundle_test.go b/e2e/support-bundle_test.go
index 79a5ddb2d..00e80fd00 100644
--- a/e2e/support-bundle_test.go
+++ b/e2e/support-bundle_test.go
@@ -24,12 +24,20 @@ func TestCollectSupportBundle(t *testing.T) {
 		t.Fatalf("fail to install embedded-cluster on node %s: %v", tc.Nodes[0], err)
 	}
 
-	line = []string{"collect-support-bundle.sh"}
+	line = []string{"collect-support-bundle-host.sh"}
 	stdout, stderr, err := RunCommandOnNode(t, tc, 0, line)
 	if err != nil {
 		t.Log("stdout:", stdout)
 		t.Log("stderr:", stderr)
-		t.Fatalf("fail to collect support bundle on node %s: %v", tc.Nodes[0], err)
+		t.Fatalf("fail to collect host support bundle on node %s: %v", tc.Nodes[0], err)
+	}
+
+	line = []string{"collect-support-bundle-cluster.sh"}
+	stdout, stderr, err = RunCommandOnNode(t, tc, 0, line)
+	if err != nil {
+		t.Log("stdout:", stdout)
+		t.Log("stderr:", stderr)
+		t.Fatalf("fail to collect  cluster support bundle on node %s: %v", tc.Nodes[0], err)
 	}
 
 	line = []string{"validate-support-bundle.sh"}

From 9486ba4d95cb070a1250376354f20db8f47f65a4 Mon Sep 17 00:00:00 2001
From: Ethan Mosbaugh <ethan@replicated.com>
Date: Wed, 10 Jul 2024 15:46:30 -0700
Subject: [PATCH 67/96] f

---
 e2e/playwright/tests/deploy-upgrade/test.spec.ts | 2 +-
 e2e/scripts/collect-support-bundle-cluster.sh    | 0
 e2e/support-bundle_test.go                       | 3 +++
 3 files changed, 4 insertions(+), 1 deletion(-)
 mode change 100644 => 100755 e2e/scripts/collect-support-bundle-cluster.sh

diff --git a/e2e/playwright/tests/deploy-upgrade/test.spec.ts b/e2e/playwright/tests/deploy-upgrade/test.spec.ts
index a94c4d4d8..dc957e928 100644
--- a/e2e/playwright/tests/deploy-upgrade/test.spec.ts
+++ b/e2e/playwright/tests/deploy-upgrade/test.spec.ts
@@ -16,7 +16,7 @@ test('deploy upgrade', async ({ page }) => {
   await iframe.getByRole('button', { name: 'Next: Confirm and deploy', exact: true }).click({ timeout: 10 * 1000 });
   await iframe.getByRole('button', { name: 'Deploy', exact: true }).click();
   await expect(page.locator('.Modal-body')).toContainText('Cluster update in progress');
-  await expect(page.locator('.Modal-body').getByText('Cluster update in progress')).not.toBeVisible({ timeout: 5 * 60 * 1000 });
+  await expect(page.locator('.Modal-body').getByText('Cluster update in progress')).not.toBeVisible({ timeout: 10 * 60 * 1000 });
   await page.getByRole('link', { name: 'Dashboard', exact: true }).click();
   await expect(page.locator('.VersionCard-content--wrapper')).toContainText(process.env.APP_UPGRADE_VERSION);
   await expect(page.locator('#app')).toContainText('Currently deployed version', { timeout: 2 * 60 * 1000 });
diff --git a/e2e/scripts/collect-support-bundle-cluster.sh b/e2e/scripts/collect-support-bundle-cluster.sh
old mode 100644
new mode 100755
diff --git a/e2e/support-bundle_test.go b/e2e/support-bundle_test.go
index 00e80fd00..d0afecd77 100644
--- a/e2e/support-bundle_test.go
+++ b/e2e/support-bundle_test.go
@@ -9,6 +9,9 @@ import (
 
 func TestCollectSupportBundle(t *testing.T) {
 	t.Parallel()
+
+	RequireEnvVars(t, []string{"SHORT_SHA"})
+
 	tc := cluster.NewTestCluster(&cluster.Input{
 		T:                   t,
 		Nodes:               1,

From 0ffe8e4fbd842f8cafc77c7d22edd67b8f469453 Mon Sep 17 00:00:00 2001
From: Andrew Lavery <laverya@umich.edu>
Date: Thu, 11 Jul 2024 18:22:17 -0400
Subject: [PATCH 68/96] add k0s images dir to host support bundle collector

---
 pkg/goods/support/host-support-bundle.yaml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/pkg/goods/support/host-support-bundle.yaml b/pkg/goods/support/host-support-bundle.yaml
index 5402831d1..77b90c09b 100644
--- a/pkg/goods/support/host-support-bundle.yaml
+++ b/pkg/goods/support/host-support-bundle.yaml
@@ -40,6 +40,10 @@ spec:
       collectorName: k0scontroller-logs
       command: journalctl
       args: [ "--since", "2 days ago", "--no-pager", "-u", "k0scontroller.service" ]
+  - run:
+      collectorName: k0s-images-dir
+      command: ls
+      args: [ "-alh", "/var/lib/k0s/images" ]
   hostAnalyzers:
   - ipv4Interfaces:
       outcomes:

From db700cb8aee61055c873a6a2535d702171ce3969 Mon Sep 17 00:00:00 2001
From: Andrew Lavery <laverya@umich.edu>
Date: Thu, 11 Jul 2024 19:11:50 -0400
Subject: [PATCH 69/96] bump previous k0s version to be the latest 1.28 patch

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 2ee5e3274..1095564e1 100644
--- a/Makefile
+++ b/Makefile
@@ -22,7 +22,7 @@ VELERO_AWS_PLUGIN_IMAGE_VERSION = v1.9.2
 KUBECTL_VERSION = v1.30.1
 K0S_VERSION = v1.29.5+k0s.0-ec.0
 K0S_GO_VERSION = v1.29.5+k0s.0
-PREVIOUS_K0S_VERSION ?= v1.28.8+k0s.0
+PREVIOUS_K0S_VERSION ?= v1.28.10+k0s.0
 K0S_BINARY_SOURCE_OVERRIDE = https://ec-k0s-binaries.s3.amazonaws.com/k0s-v1.29.5%2Bk0s.0-ec.0
 PREVIOUS_K0S_BINARY_SOURCE_OVERRIDE =
 TROUBLESHOOT_VERSION = v0.93.1

From 28e711dc1b568054a86652c78367da4cf19dac9d Mon Sep 17 00:00:00 2001
From: Andrew Lavery <laverya@umich.edu>
Date: Fri, 12 Jul 2024 09:26:01 -0400
Subject: [PATCH 70/96] second to latest 1.28 patch

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 1095564e1..d90cc5eca 100644
--- a/Makefile
+++ b/Makefile
@@ -22,7 +22,7 @@ VELERO_AWS_PLUGIN_IMAGE_VERSION = v1.9.2
 KUBECTL_VERSION = v1.30.1
 K0S_VERSION = v1.29.5+k0s.0-ec.0
 K0S_GO_VERSION = v1.29.5+k0s.0
-PREVIOUS_K0S_VERSION ?= v1.28.10+k0s.0
+PREVIOUS_K0S_VERSION ?= v1.28.9+k0s.0
 K0S_BINARY_SOURCE_OVERRIDE = https://ec-k0s-binaries.s3.amazonaws.com/k0s-v1.29.5%2Bk0s.0-ec.0
 PREVIOUS_K0S_BINARY_SOURCE_OVERRIDE =
 TROUBLESHOOT_VERSION = v0.93.1

From f14edac701afbbc59c952397dae9d01fd7675324 Mon Sep 17 00:00:00 2001
From: Andrew Lavery <laverya@umich.edu>
Date: Fri, 12 Jul 2024 10:24:29 -0400
Subject: [PATCH 71/96] back to 1.28.10

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index d90cc5eca..1095564e1 100644
--- a/Makefile
+++ b/Makefile
@@ -22,7 +22,7 @@ VELERO_AWS_PLUGIN_IMAGE_VERSION = v1.9.2
 KUBECTL_VERSION = v1.30.1
 K0S_VERSION = v1.29.5+k0s.0-ec.0
 K0S_GO_VERSION = v1.29.5+k0s.0
-PREVIOUS_K0S_VERSION ?= v1.28.9+k0s.0
+PREVIOUS_K0S_VERSION ?= v1.28.10+k0s.0
 K0S_BINARY_SOURCE_OVERRIDE = https://ec-k0s-binaries.s3.amazonaws.com/k0s-v1.29.5%2Bk0s.0-ec.0
 PREVIOUS_K0S_BINARY_SOURCE_OVERRIDE =
 TROUBLESHOOT_VERSION = v0.93.1

From 22e8b9b954c3661adfc32c5f154968c110d01493 Mon Sep 17 00:00:00 2001
From: Andrew Lavery <laverya@umich.edu>
Date: Fri, 12 Jul 2024 12:19:54 -0400
Subject: [PATCH 72/96] test reset and reinstall

---
 .github/workflows/pull-request.yaml | 84 ++++++++++++++---------------
 e2e/install_test.go                 |  4 +-
 e2e/scripts/single-node-install.sh  |  7 +++
 3 files changed, 52 insertions(+), 43 deletions(-)

diff --git a/.github/workflows/pull-request.yaml b/.github/workflows/pull-request.yaml
index d389d7742..eb9473388 100644
--- a/.github/workflows/pull-request.yaml
+++ b/.github/workflows/pull-request.yaml
@@ -199,50 +199,50 @@ jobs:
       fail-fast: false
       matrix:
         test:
-          - TestSingleNodeInstallation
-          - TestSingleNodeInstallationAlmaLinux8
-          - TestSingleNodeInstallationDebian11
-          - TestSingleNodeInstallationDebian12
-          - TestSingleNodeInstallationCentos9Stream
-          - TestVersion
-          - TestHostPreflight
-          - TestUnsupportedOverrides
-          - TestMultiNodeInstallation
-          - TestMultiNodeReset
-          - TestCommandsRequireSudo
-          - TestInstallWithoutEmbed
-          - TestInstallFromReplicatedApp
+#          - TestSingleNodeInstallation
+#          - TestSingleNodeInstallationAlmaLinux8
+#          - TestSingleNodeInstallationDebian11
+#          - TestSingleNodeInstallationDebian12
+#          - TestSingleNodeInstallationCentos9Stream
+#          - TestVersion
+#          - TestHostPreflight
+#          - TestUnsupportedOverrides
+#          - TestMultiNodeInstallation
+#          - TestMultiNodeReset
+#          - TestCommandsRequireSudo
+#          - TestInstallWithoutEmbed
+#          - TestInstallFromReplicatedApp
           - TestResetAndReinstall
           - TestResetAndReinstallAirgap
-          - TestCollectSupportBundle
-          - TestOldVersionUpgrade
-          - TestMaterialize
-          - TestLocalArtifactMirror
-          - TestSingleNodeAirgapUpgrade
-          - TestSingleNodeAirgapUpgradeCustomCIDR
-          - TestInstallSnapshotFromReplicatedApp
-          - TestMultiNodeAirgapUpgrade
-          - TestSingleNodeDisasterRecovery
-          - TestSingleNodeDisasterRecoveryWithProxy
-          - TestSingleNodeResumeDisasterRecovery
-          - TestSingleNodeAirgapDisasterRecovery
-          - TestMultiNodeHAInstallation
-          - TestMultiNodeAirgapHAInstallation
-          - TestMultiNodeAirgapUpgradeSameK0s
-          - TestProxiedEnvironment
-          - TestMultiNodeHADisasterRecovery
-          - TestMultiNodeAirgapHADisasterRecovery
-          - TestCustomCIDR
-          - TestProxiedCustomCIDR
-        include:
-          - test: TestMultiNodeAirgapUpgrade
-            runner: embedded-cluster
-          - test: TestMultiNodeAirgapHAInstallation
-            runner: embedded-cluster
-          - test: TestMultiNodeAirgapHADisasterRecovery
-            runner: embedded-cluster
-          - test: TestSingleNodeAirgapDisasterRecovery
-            runner: embedded-cluster
+#          - TestCollectSupportBundle
+#          - TestOldVersionUpgrade
+#          - TestMaterialize
+#          - TestLocalArtifactMirror
+#          - TestSingleNodeAirgapUpgrade
+#          - TestSingleNodeAirgapUpgradeCustomCIDR
+#          - TestInstallSnapshotFromReplicatedApp
+#          - TestMultiNodeAirgapUpgrade
+#          - TestSingleNodeDisasterRecovery
+#          - TestSingleNodeDisasterRecoveryWithProxy
+#          - TestSingleNodeResumeDisasterRecovery
+#          - TestSingleNodeAirgapDisasterRecovery
+#          - TestMultiNodeHAInstallation
+#          - TestMultiNodeAirgapHAInstallation
+#          - TestMultiNodeAirgapUpgradeSameK0s
+#          - TestProxiedEnvironment
+#          - TestMultiNodeHADisasterRecovery
+#          - TestMultiNodeAirgapHADisasterRecovery
+#          - TestCustomCIDR
+#          - TestProxiedCustomCIDR
+#        include:
+#          - test: TestMultiNodeAirgapUpgrade
+#            runner: embedded-cluster
+#          - test: TestMultiNodeAirgapHAInstallation
+#            runner: embedded-cluster
+#          - test: TestMultiNodeAirgapHADisasterRecovery
+#            runner: embedded-cluster
+#          - test: TestSingleNodeAirgapDisasterRecovery
+#            runner: embedded-cluster
     steps:
     - name: Checkout
       uses: actions/checkout@v4
diff --git a/e2e/install_test.go b/e2e/install_test.go
index dd131b27e..48d506160 100644
--- a/e2e/install_test.go
+++ b/e2e/install_test.go
@@ -525,9 +525,11 @@ func TestResetAndReinstall(t *testing.T) {
 
 	t.Logf("%s: installing embedded-cluster on node 0 after reset", time.Now().Format(time.RFC3339))
 	line = []string{"single-node-install.sh", "ui"}
-	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
+	stdout, _, err := RunCommandOnNode(t, tc, 0, line)
+	if err != nil {
 		t.Fatalf("fail to install embedded-cluster on node %s: %v", tc.Nodes[0], err)
 	}
+	t.Log(stdout)
 
 	if _, _, err := runPlaywrightTest(t, tc, "deploy-app"); err != nil {
 		t.Fatalf("fail to run playwright test deploy-app: %v", err)
diff --git a/e2e/scripts/single-node-install.sh b/e2e/scripts/single-node-install.sh
index 818465c81..7be40c869 100755
--- a/e2e/scripts/single-node-install.sh
+++ b/e2e/scripts/single-node-install.sh
@@ -340,6 +340,13 @@ main() {
         echo "Failed to get status of embedded-cluster service"
         exit 1
     fi
+
+    echo "kotsadm logs"
+    kubectl logs -n kotsadm -l app=kotsadm --tail=50
+    kubectl logs -n kotsadm -l app=kotsadm --tail=50 --previous
+
+    echo "all pods"
+    kubectl get pods -A
 }
 
 export EMBEDDED_CLUSTER_METRICS_BASEURL="https://staging.replicated.app"

From f88c16b34f9779e5bd3479240487a3e1ab45da22 Mon Sep 17 00:00:00 2001
From: Andrew Lavery <laverya@umich.edu>
Date: Fri, 12 Jul 2024 12:39:09 -0400
Subject: [PATCH 73/96] f

---
 e2e/scripts/single-node-install.sh | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/e2e/scripts/single-node-install.sh b/e2e/scripts/single-node-install.sh
index 7be40c869..54c99c5b1 100755
--- a/e2e/scripts/single-node-install.sh
+++ b/e2e/scripts/single-node-install.sh
@@ -342,8 +342,9 @@ main() {
     fi
 
     echo "kotsadm logs"
-    kubectl logs -n kotsadm -l app=kotsadm --tail=50
-    kubectl logs -n kotsadm -l app=kotsadm --tail=50 --previous
+    kubectl logs -n kotsadm -l app=kotsadm --tail=50 || true
+    echo "previous kotsadm logs"
+    kubectl logs -n kotsadm -l app=kotsadm --tail=50 --previous || true
 
     echo "all pods"
     kubectl get pods -A

From 6aa6d148a2e6d12d6236e6babe4684f1c7f3b5d4 Mon Sep 17 00:00:00 2001
From: Andrew Lavery <laverya@umich.edu>
Date: Fri, 12 Jul 2024 13:06:45 -0400
Subject: [PATCH 74/96] call playwright setup again after resetting instance

---
 e2e/install_test.go | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/e2e/install_test.go b/e2e/install_test.go
index 48d506160..57ab5a80c 100644
--- a/e2e/install_test.go
+++ b/e2e/install_test.go
@@ -525,12 +525,13 @@ func TestResetAndReinstall(t *testing.T) {
 
 	t.Logf("%s: installing embedded-cluster on node 0 after reset", time.Now().Format(time.RFC3339))
 	line = []string{"single-node-install.sh", "ui"}
-	stdout, _, err := RunCommandOnNode(t, tc, 0, line)
-	if err != nil {
+	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
 		t.Fatalf("fail to install embedded-cluster on node %s: %v", tc.Nodes[0], err)
 	}
-	t.Log(stdout)
 
+	if err := setupPlaywright(t, tc); err != nil {
+		t.Fatalf("fail to setup playwright: %v", err)
+	}
 	if _, _, err := runPlaywrightTest(t, tc, "deploy-app"); err != nil {
 		t.Fatalf("fail to run playwright test deploy-app: %v", err)
 	}

From c8ad5bdf0f408ad90cfb2695bfaf230c26021514 Mon Sep 17 00:00:00 2001
From: Andrew Lavery <laverya@umich.edu>
Date: Fri, 12 Jul 2024 13:29:23 -0400
Subject: [PATCH 75/96] reenable all tests

---
 .github/workflows/pull-request.yaml | 84 ++++++++++++++---------------
 1 file changed, 42 insertions(+), 42 deletions(-)

diff --git a/.github/workflows/pull-request.yaml b/.github/workflows/pull-request.yaml
index eb9473388..d389d7742 100644
--- a/.github/workflows/pull-request.yaml
+++ b/.github/workflows/pull-request.yaml
@@ -199,50 +199,50 @@ jobs:
       fail-fast: false
       matrix:
         test:
-#          - TestSingleNodeInstallation
-#          - TestSingleNodeInstallationAlmaLinux8
-#          - TestSingleNodeInstallationDebian11
-#          - TestSingleNodeInstallationDebian12
-#          - TestSingleNodeInstallationCentos9Stream
-#          - TestVersion
-#          - TestHostPreflight
-#          - TestUnsupportedOverrides
-#          - TestMultiNodeInstallation
-#          - TestMultiNodeReset
-#          - TestCommandsRequireSudo
-#          - TestInstallWithoutEmbed
-#          - TestInstallFromReplicatedApp
+          - TestSingleNodeInstallation
+          - TestSingleNodeInstallationAlmaLinux8
+          - TestSingleNodeInstallationDebian11
+          - TestSingleNodeInstallationDebian12
+          - TestSingleNodeInstallationCentos9Stream
+          - TestVersion
+          - TestHostPreflight
+          - TestUnsupportedOverrides
+          - TestMultiNodeInstallation
+          - TestMultiNodeReset
+          - TestCommandsRequireSudo
+          - TestInstallWithoutEmbed
+          - TestInstallFromReplicatedApp
           - TestResetAndReinstall
           - TestResetAndReinstallAirgap
-#          - TestCollectSupportBundle
-#          - TestOldVersionUpgrade
-#          - TestMaterialize
-#          - TestLocalArtifactMirror
-#          - TestSingleNodeAirgapUpgrade
-#          - TestSingleNodeAirgapUpgradeCustomCIDR
-#          - TestInstallSnapshotFromReplicatedApp
-#          - TestMultiNodeAirgapUpgrade
-#          - TestSingleNodeDisasterRecovery
-#          - TestSingleNodeDisasterRecoveryWithProxy
-#          - TestSingleNodeResumeDisasterRecovery
-#          - TestSingleNodeAirgapDisasterRecovery
-#          - TestMultiNodeHAInstallation
-#          - TestMultiNodeAirgapHAInstallation
-#          - TestMultiNodeAirgapUpgradeSameK0s
-#          - TestProxiedEnvironment
-#          - TestMultiNodeHADisasterRecovery
-#          - TestMultiNodeAirgapHADisasterRecovery
-#          - TestCustomCIDR
-#          - TestProxiedCustomCIDR
-#        include:
-#          - test: TestMultiNodeAirgapUpgrade
-#            runner: embedded-cluster
-#          - test: TestMultiNodeAirgapHAInstallation
-#            runner: embedded-cluster
-#          - test: TestMultiNodeAirgapHADisasterRecovery
-#            runner: embedded-cluster
-#          - test: TestSingleNodeAirgapDisasterRecovery
-#            runner: embedded-cluster
+          - TestCollectSupportBundle
+          - TestOldVersionUpgrade
+          - TestMaterialize
+          - TestLocalArtifactMirror
+          - TestSingleNodeAirgapUpgrade
+          - TestSingleNodeAirgapUpgradeCustomCIDR
+          - TestInstallSnapshotFromReplicatedApp
+          - TestMultiNodeAirgapUpgrade
+          - TestSingleNodeDisasterRecovery
+          - TestSingleNodeDisasterRecoveryWithProxy
+          - TestSingleNodeResumeDisasterRecovery
+          - TestSingleNodeAirgapDisasterRecovery
+          - TestMultiNodeHAInstallation
+          - TestMultiNodeAirgapHAInstallation
+          - TestMultiNodeAirgapUpgradeSameK0s
+          - TestProxiedEnvironment
+          - TestMultiNodeHADisasterRecovery
+          - TestMultiNodeAirgapHADisasterRecovery
+          - TestCustomCIDR
+          - TestProxiedCustomCIDR
+        include:
+          - test: TestMultiNodeAirgapUpgrade
+            runner: embedded-cluster
+          - test: TestMultiNodeAirgapHAInstallation
+            runner: embedded-cluster
+          - test: TestMultiNodeAirgapHADisasterRecovery
+            runner: embedded-cluster
+          - test: TestSingleNodeAirgapDisasterRecovery
+            runner: embedded-cluster
     steps:
     - name: Checkout
       uses: actions/checkout@v4

From bcbd7b36a69946c09071d475198e9739a64325c6 Mon Sep 17 00:00:00 2001
From: Andrew Lavery <laverya@umich.edu>
Date: Fri, 12 Jul 2024 13:40:49 -0400
Subject: [PATCH 76/96] add TestSingleNodeInstallationNoopUpgrade test

---
 .github/workflows/pull-request.yaml |  2 ++
 .github/workflows/release-dev.yaml  |  2 ++
 e2e/install_test.go                 | 50 +++++++++++++++++++++++++++++
 3 files changed, 54 insertions(+)

diff --git a/.github/workflows/pull-request.yaml b/.github/workflows/pull-request.yaml
index d389d7742..42ec5295e 100644
--- a/.github/workflows/pull-request.yaml
+++ b/.github/workflows/pull-request.yaml
@@ -110,6 +110,7 @@ jobs:
         replicated release promote 807 2cHXb1RCttzpR0xvnNWyaZCgDBP --version "appver-${SHORT_SHA}-pre-minio-removal"
 
         replicated release create --yaml-dir e2e/kots-release-install --promote CI --version "appver-${SHORT_SHA}"
+        replicated release create --yaml-dir e2e/kots-release-install --promote CI --version "appver-${SHORT_SHA}-noop"
         replicated release create --yaml-dir e2e/kots-release-upgrade --promote CI --version "appver-${SHORT_SHA}-upgrade"
     - name: Build CI binary
       run: |
@@ -234,6 +235,7 @@ jobs:
           - TestMultiNodeAirgapHADisasterRecovery
           - TestCustomCIDR
           - TestProxiedCustomCIDR
+          - TestSingleNodeInstallationNoopUpgrade
         include:
           - test: TestMultiNodeAirgapUpgrade
             runner: embedded-cluster
diff --git a/.github/workflows/release-dev.yaml b/.github/workflows/release-dev.yaml
index bfd939c9d..1aa07c2e1 100644
--- a/.github/workflows/release-dev.yaml
+++ b/.github/workflows/release-dev.yaml
@@ -83,6 +83,7 @@ jobs:
           replicated release promote 807 2cHXb1RCttzpR0xvnNWyaZCgDBP --version "appver-${SHORT_SHA}-pre-minio-removal"
           
           replicated release create --yaml-dir e2e/kots-release-install --promote CI --version "appver-${SHORT_SHA}"
+          replicated release create --yaml-dir e2e/kots-release-install --promote CI --version "appver-${SHORT_SHA}-noop"
           replicated release create --yaml-dir e2e/kots-release-upgrade --promote CI --version "appver-${SHORT_SHA}-upgrade"
       - name: Build CI binary
         run: |
@@ -179,6 +180,7 @@ jobs:
           - TestMultiNodeAirgapHADisasterRecovery
           - TestCustomCIDR
           - TestProxiedCustomCIDR
+          - TestSingleNodeInstallationNoopUpgrade
         include:
           - test: TestMultiNodeAirgapUpgrade
             runner: embedded-cluster
diff --git a/e2e/install_test.go b/e2e/install_test.go
index 57ab5a80c..287051a7c 100644
--- a/e2e/install_test.go
+++ b/e2e/install_test.go
@@ -1668,6 +1668,56 @@ func TestCustomCIDR(t *testing.T) {
 	t.Logf("%s: test complete", time.Now().Format(time.RFC3339))
 }
 
+func TestSingleNodeInstallationNoopUpgrade(t *testing.T) {
+	t.Parallel()
+
+	RequireEnvVars(t, []string{"SHORT_SHA"})
+
+	tc := cluster.NewTestCluster(&cluster.Input{
+		T:                   t,
+		Nodes:               1,
+		Image:               "ubuntu/jammy",
+		LicensePath:         "license.yaml",
+		EmbeddedClusterPath: "../output/bin/embedded-cluster",
+	})
+	defer cleanupCluster(t, tc)
+
+	t.Logf("%s: installing embedded-cluster on node 0", time.Now().Format(time.RFC3339))
+	line := []string{"single-node-install.sh", "ui"}
+	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
+		t.Fatalf("fail to install embedded-cluster on node %s: %v", tc.Nodes[0], err)
+	}
+
+	if err := setupPlaywright(t, tc); err != nil {
+		t.Fatalf("fail to setup playwright: %v", err)
+	}
+	if _, _, err := runPlaywrightTest(t, tc, "deploy-app"); err != nil {
+		t.Fatalf("fail to run playwright test deploy-app: %v", err)
+	}
+
+	t.Logf("%s: checking installation state", time.Now().Format(time.RFC3339))
+	line = []string{"check-installation-state.sh", os.Getenv("SHORT_SHA")}
+	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
+		t.Fatalf("fail to check installation state: %v", err)
+	}
+
+	appUpgradeVersion := fmt.Sprintf("appver-%s-noop", os.Getenv("SHORT_SHA"))
+	testArgs := []string{appUpgradeVersion}
+
+	t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339))
+	if _, _, err := runPlaywrightTest(t, tc, "deploy-upgrade", testArgs...); err != nil {
+		t.Fatalf("fail to run playwright test deploy-app: %v", err)
+	}
+
+	t.Logf("%s: checking installation state after noop upgrade", time.Now().Format(time.RFC3339))
+	line = []string{"check-installation-state.sh", os.Getenv("SHORT_SHA")}
+	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
+		t.Fatalf("fail to check installation state: %v", err)
+	}
+
+	t.Logf("%s: test complete", time.Now().Format(time.RFC3339))
+}
+
 func downloadAirgapBundle(t *testing.T, versionLabel string, destPath string, licenseID string) string {
 	// download airgap bundle
 	airgapURL := fmt.Sprintf("https://staging.replicated.app/embedded/embedded-cluster-smoke-test-staging-app/ci-airgap/%s?airgap=true", versionLabel)

From 61efe0ea330623588744b63cdbeab8a545f651f4 Mon Sep 17 00:00:00 2001
From: Andrew Lavery <laverya@umich.edu>
Date: Fri, 12 Jul 2024 14:10:03 -0400
Subject: [PATCH 77/96] operator image override

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 1095564e1..b8c073c3e 100644
--- a/Makefile
+++ b/Makefile
@@ -10,7 +10,7 @@ ADMIN_CONSOLE_KURL_PROXY_IMAGE_OVERRIDE = kotsadm/kurl-proxy:v2024.7.8-ec8-night
 EMBEDDED_OPERATOR_CHART_VERSION = 0.39.0-build.3
 EMBEDDED_OPERATOR_BINARY_URL_OVERRIDE =
 EMBEDDED_OPERATOR_UTILS_IMAGE = busybox:1.36.1
-EMBEDDED_CLUSTER_OPERATOR_IMAGE_OVERRIDE =
+EMBEDDED_CLUSTER_OPERATOR_IMAGE_OVERRIDE = ttl.sh/embedded-cluster-operator-image:dev-a11a481
 OPENEBS_CHART_VERSION = 4.1.0
 OPENEBS_UTILS_VERSION = 4.1.0
 SEAWEEDFS_CHART_VERSION = 4.0.0

From 94ce10b68f61971ff9374eea0a320617bbb972d9 Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Fri, 12 Jul 2024 11:26:55 -0700
Subject: [PATCH 78/96] refine playwright tests

---
 .../tests/deploy-airgap-upgrade/test.spec.ts  | 18 -----------
 .../tests/deploy-upgrade/test.spec.ts         | 30 +++++++++++++------
 e2e/playwright/tests/shared/deploy-app.ts     |  6 +++-
 3 files changed, 26 insertions(+), 28 deletions(-)
 delete mode 100644 e2e/playwright/tests/deploy-airgap-upgrade/test.spec.ts

diff --git a/e2e/playwright/tests/deploy-airgap-upgrade/test.spec.ts b/e2e/playwright/tests/deploy-airgap-upgrade/test.spec.ts
deleted file mode 100644
index 7d0c61be1..000000000
--- a/e2e/playwright/tests/deploy-airgap-upgrade/test.spec.ts
+++ /dev/null
@@ -1,18 +0,0 @@
-import { test, expect } from '@playwright/test';
-import { login } from '../shared';
-
-test('deploy airgap upgrade', async ({ page }) => {
-  test.setTimeout(15 * 60 * 1000); // 15 minutes
-  await login(page);
-  await expect(page.locator('#app')).toContainText('Airgap Update');
-  await page.getByRole('button', { name: 'Deploy', exact: true }).click();
-  await expect(page.locator('.Modal-body')).toBeVisible();
-  await page.getByRole('button', { name: 'Yes, Deploy' }).click();
-  await expect(page.locator('#app')).toContainText('Updating cluster', { timeout: 90000 });
-  if (process.env.SKIP_CLUSTER_UPGRADING_CHECK !== 'true') {
-    await expect(page.locator('.Modal-body')).toContainText('Cluster update in progress', { timeout: 120000 });
-  }
-  await expect(page.locator('#app')).toContainText('Currently deployed version', { timeout: 600000 });
-  await expect(page.locator('#app')).toContainText('Up to date', { timeout: 30000 });
-  await expect(page.locator('#app')).toContainText('Ready');
-});
diff --git a/e2e/playwright/tests/deploy-upgrade/test.spec.ts b/e2e/playwright/tests/deploy-upgrade/test.spec.ts
index dc957e928..182aa1ef1 100644
--- a/e2e/playwright/tests/deploy-upgrade/test.spec.ts
+++ b/e2e/playwright/tests/deploy-upgrade/test.spec.ts
@@ -5,21 +5,33 @@ test('deploy upgrade', async ({ page }) => {
   test.setTimeout(15 * 60 * 1000); // 15 minutes
   await login(page);
   await page.getByRole('link', { name: 'Version history', exact: true }).click();
-  const rowLocator = '../../..';
-  // const rowLocator = '//./ancestor::div[@class="available-update-row"]';
-  await page.getByText(process.env.APP_UPGRADE_VERSION, { exact: true })
-    .locator(rowLocator)
-    .getByRole('button', { name: 'Deploy', exact: true }).click();
+  await page.locator('.available-update-row', { hasText: process.env.APP_UPGRADE_VERSION }).getByRole('button', { name: 'Deploy', exact: true }).click();
   const iframe = page.frameLocator('#upgrade-service-iframe');
-  await expect(iframe.locator('.ConfigArea--wrapper')).toBeVisible({ timeout: 20 * 1000 });
+  await expect(iframe.locator('h3')).toContainText('The First Config Group', { timeout: 20 * 1000 });
+  await expect(iframe.locator('input[type="text"]')).toHaveValue('initial-hostname.com');
+  await iframe.locator('input[type="text"]').click();
+  await iframe.locator('input[type="text"]').fill('updated-hostname.com');
+  await iframe.locator('input[type="password"]').click();
+  await iframe.locator('input[type="password"]').fill('updated password');
   await iframe.getByRole('button', { name: 'Next', exact: true }).click();
-  await iframe.getByRole('button', { name: 'Next: Confirm and deploy', exact: true }).click({ timeout: 10 * 1000 });
+  await expect(iframe.getByText('Preflight checks', { exact: true })).toBeVisible({ timeout: 10 * 1000 });
+  await expect(iframe.getByRole('button', { name: 'Re-run' })).toBeVisible({ timeout: 10 * 1000 });
+  await expect(iframe.locator('#app')).toContainText('Embedded Cluster Installation CRD exists');
+  await expect(iframe.locator('#app')).toContainText('Embedded Cluster Config CRD exists');
+  await expect(iframe.getByRole('button', { name: 'Back: Config' })).toBeVisible();
+  await iframe.getByRole('button', { name: 'Next: Confirm and deploy' }).click();
+  await expect(iframe.locator('#app')).toContainText('All preflight checks passed');
+  await expect(iframe.getByRole('button', { name: 'Back: Preflight checks' })).toBeVisible();
   await iframe.getByRole('button', { name: 'Deploy', exact: true }).click();
   await expect(page.locator('.Modal-body')).toContainText('Cluster update in progress');
   await expect(page.locator('.Modal-body').getByText('Cluster update in progress')).not.toBeVisible({ timeout: 10 * 60 * 1000 });
+  // wait for the page to refresh after cluster update
+  await page.waitForTimeout(5000);
+  await expect(page.locator('.available-update-row', { hasText: 'appver-dev-3f50304-upgrade' })).not.toBeVisible();
+  await expect(page.locator('.VersionHistoryRow', { hasText: 'appver-dev-3f50304' })).toContainText('Currently deployed version', { timeout: 30 * 1000 });
   await page.getByRole('link', { name: 'Dashboard', exact: true }).click();
-  await expect(page.locator('.VersionCard-content--wrapper')).toContainText(process.env.APP_UPGRADE_VERSION);
-  await expect(page.locator('#app')).toContainText('Currently deployed version', { timeout: 2 * 60 * 1000 });
+  await expect(page.locator('.VersionCard-content--wrapper')).toContainText('appver-dev-3f50304');
+  await expect(page.locator('#app')).toContainText('Currently deployed version');
   await expect(page.locator('#app')).toContainText('Ready', { timeout: 30 * 1000 });
   await expect(page.locator('#app')).toContainText('Up to date');
 });
diff --git a/e2e/playwright/tests/shared/deploy-app.ts b/e2e/playwright/tests/shared/deploy-app.ts
index 72ff20eab..80c2eed58 100644
--- a/e2e/playwright/tests/shared/deploy-app.ts
+++ b/e2e/playwright/tests/shared/deploy-app.ts
@@ -3,10 +3,14 @@ export const deployApp = async (page, expect) => {
   await page.getByRole('button', { name: 'Continue' }).click();
   await expect(page.locator('h3')).toContainText('The First Config Group');
   await page.locator('input[type="text"]').click();
-  await page.locator('input[type="text"]').fill('abc');
+  await page.locator('input[type="text"]').fill('initial-hostname.com');
   await page.locator('input[type="password"]').click();
   await page.locator('input[type="password"]').fill('password');
   await page.getByRole('button', { name: 'Continue' }).click();
+  await expect(page.getByText('Preflight checks', { exact: true })).toBeVisible({ timeout: 10 * 1000 });
+  await expect(page.getByRole('button', { name: 'Re-run' })).toBeVisible({ timeout: 10 * 1000 });
+  await expect(page.locator('#app')).toContainText('Embedded Cluster Installation CRD exists');
+  await expect(page.locator('#app')).toContainText('Embedded Cluster Config CRD exists');
   await page.getByRole('button', { name: 'Deploy' }).click();
   await expect(page.locator('#app')).toContainText('Currently deployed version', { timeout: 90000 });
   await expect(page.locator('#app')).toContainText('Ready', { timeout: 30000 });

From b6c0471a5d2ad3072d1013f8f8fb798e56c3d344 Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Fri, 12 Jul 2024 11:28:00 -0700
Subject: [PATCH 79/96] use version from env var

---
 e2e/playwright/tests/deploy-upgrade/test.spec.ts | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/e2e/playwright/tests/deploy-upgrade/test.spec.ts b/e2e/playwright/tests/deploy-upgrade/test.spec.ts
index 182aa1ef1..7f39dfb65 100644
--- a/e2e/playwright/tests/deploy-upgrade/test.spec.ts
+++ b/e2e/playwright/tests/deploy-upgrade/test.spec.ts
@@ -27,10 +27,10 @@ test('deploy upgrade', async ({ page }) => {
   await expect(page.locator('.Modal-body').getByText('Cluster update in progress')).not.toBeVisible({ timeout: 10 * 60 * 1000 });
   // wait for the page to refresh after cluster update
   await page.waitForTimeout(5000);
-  await expect(page.locator('.available-update-row', { hasText: 'appver-dev-3f50304-upgrade' })).not.toBeVisible();
-  await expect(page.locator('.VersionHistoryRow', { hasText: 'appver-dev-3f50304' })).toContainText('Currently deployed version', { timeout: 30 * 1000 });
+  await expect(page.locator('.available-update-row', { hasText: process.env.APP_UPGRADE_VERSION })).not.toBeVisible();
+  await expect(page.locator('.VersionHistoryRow', { hasText: process.env.APP_UPGRADE_VERSION })).toContainText('Currently deployed version', { timeout: 30 * 1000 });
   await page.getByRole('link', { name: 'Dashboard', exact: true }).click();
-  await expect(page.locator('.VersionCard-content--wrapper')).toContainText('appver-dev-3f50304');
+  await expect(page.locator('.VersionCard-content--wrapper')).toContainText(process.env.APP_UPGRADE_VERSION);
   await expect(page.locator('#app')).toContainText('Currently deployed version');
   await expect(page.locator('#app')).toContainText('Ready', { timeout: 30 * 1000 });
   await expect(page.locator('#app')).toContainText('Up to date');

From 9b88c558d0c4bc6fbc8e3376949d1603c3eedf1a Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Fri, 12 Jul 2024 13:11:41 -0700
Subject: [PATCH 80/96] skip cluster update modal for noop test

---
 .../tests/deploy-upgrade/test.spec.ts           | 17 +++++++++++------
 e2e/scripts/playwright.sh                       |  2 +-
 2 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/e2e/playwright/tests/deploy-upgrade/test.spec.ts b/e2e/playwright/tests/deploy-upgrade/test.spec.ts
index 7f39dfb65..a860b15a0 100644
--- a/e2e/playwright/tests/deploy-upgrade/test.spec.ts
+++ b/e2e/playwright/tests/deploy-upgrade/test.spec.ts
@@ -5,7 +5,10 @@ test('deploy upgrade', async ({ page }) => {
   test.setTimeout(15 * 60 * 1000); // 15 minutes
   await login(page);
   await page.getByRole('link', { name: 'Version history', exact: true }).click();
-  await page.locator('.available-update-row', { hasText: process.env.APP_UPGRADE_VERSION }).getByRole('button', { name: 'Deploy', exact: true }).click();
+  // TODO NOW: uncomment this
+  // await page.locator('.available-update-row', { hasText: process.env.APP_UPGRADE_VERSION }).getByRole('button', { name: 'Deploy', exact: true }).click();
+  // TODO NOW: remove this line
+  await page.getByText(process.env.APP_UPGRADE_VERSION, { exact: true }).locator('../../..').getByRole('button', { name: 'Deploy', exact: true }).click();
   const iframe = page.frameLocator('#upgrade-service-iframe');
   await expect(iframe.locator('h3')).toContainText('The First Config Group', { timeout: 20 * 1000 });
   await expect(iframe.locator('input[type="text"]')).toHaveValue('initial-hostname.com');
@@ -23,11 +26,13 @@ test('deploy upgrade', async ({ page }) => {
   await expect(iframe.locator('#app')).toContainText('All preflight checks passed');
   await expect(iframe.getByRole('button', { name: 'Back: Preflight checks' })).toBeVisible();
   await iframe.getByRole('button', { name: 'Deploy', exact: true }).click();
-  await expect(page.locator('.Modal-body')).toContainText('Cluster update in progress');
-  await expect(page.locator('.Modal-body').getByText('Cluster update in progress')).not.toBeVisible({ timeout: 10 * 60 * 1000 });
-  // wait for the page to refresh after cluster update
-  await page.waitForTimeout(5000);
-  await expect(page.locator('.available-update-row', { hasText: process.env.APP_UPGRADE_VERSION })).not.toBeVisible();
+
+  if (process.env.SKIP_CLUSTER_UPGRADE_CHECK !== 'true') {
+    await expect(page.locator('.Modal-body')).toContainText('Cluster update in progress');
+    await expect(page.locator('.Modal-body').getByText('Cluster update in progress')).not.toBeVisible({ timeout: 10 * 60 * 1000 });
+  }
+
+  await expect(page.locator('.available-update-row', { hasText: process.env.APP_UPGRADE_VERSION })).not.toBeVisible({ timeout: 10 * 1000 });
   await expect(page.locator('.VersionHistoryRow', { hasText: process.env.APP_UPGRADE_VERSION })).toContainText('Currently deployed version', { timeout: 30 * 1000 });
   await page.getByRole('link', { name: 'Dashboard', exact: true }).click();
   await expect(page.locator('.VersionCard-content--wrapper')).toContainText(process.env.APP_UPGRADE_VERSION);
diff --git a/e2e/scripts/playwright.sh b/e2e/scripts/playwright.sh
index 491970624..768586134 100755
--- a/e2e/scripts/playwright.sh
+++ b/e2e/scripts/playwright.sh
@@ -17,9 +17,9 @@ main() {
     export DR_AWS_SECRET_ACCESS_KEY="$7"
   elif [ "$test_name" == "deploy-upgrade" ]; then
     export APP_UPGRADE_VERSION="$2"
+    export SKIP_CLUSTER_UPGRADE_CHECK="$3"
   fi
 
-
   export BASE_URL="http://10.0.0.2:30001"
   cd /automation/playwright
   npx playwright test "$test_name"

From 53148e42779955d9022be63f7153ba35ae718a26 Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Fri, 12 Jul 2024 13:12:22 -0700
Subject: [PATCH 81/96] skip cluster update modal for noop test

---
 e2e/install_test.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/e2e/install_test.go b/e2e/install_test.go
index 287051a7c..fe64b1277 100644
--- a/e2e/install_test.go
+++ b/e2e/install_test.go
@@ -1702,7 +1702,8 @@ func TestSingleNodeInstallationNoopUpgrade(t *testing.T) {
 	}
 
 	appUpgradeVersion := fmt.Sprintf("appver-%s-noop", os.Getenv("SHORT_SHA"))
-	testArgs := []string{appUpgradeVersion}
+	skipClusterUpgradeCheck := "true"
+	testArgs := []string{appUpgradeVersion, skipClusterUpgradeCheck}
 
 	t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339))
 	if _, _, err := runPlaywrightTest(t, tc, "deploy-upgrade", testArgs...); err != nil {

From e1df3d5484431c42ce7b49de611107d915b75945 Mon Sep 17 00:00:00 2001
From: Andrew Lavery <laverya@umich.edu>
Date: Fri, 12 Jul 2024 17:59:31 -0400
Subject: [PATCH 82/96] handle unbound SKIP_CLUSTER_UPGRADE_CHECK variable

---
 e2e/scripts/playwright.sh | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/e2e/scripts/playwright.sh b/e2e/scripts/playwright.sh
index 768586134..376c8d51d 100755
--- a/e2e/scripts/playwright.sh
+++ b/e2e/scripts/playwright.sh
@@ -17,7 +17,9 @@ main() {
     export DR_AWS_SECRET_ACCESS_KEY="$7"
   elif [ "$test_name" == "deploy-upgrade" ]; then
     export APP_UPGRADE_VERSION="$2"
-    export SKIP_CLUSTER_UPGRADE_CHECK="$3"
+    if [ -n "$3" ]; then
+      export SKIP_CLUSTER_UPGRADE_CHECK="$3"
+    fi
   fi
 
   export BASE_URL="http://10.0.0.2:30001"

From 6d581eb8f3b6efc4cddb9d65344ecfccf9e3d493 Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Fri, 12 Jul 2024 15:26:55 -0700
Subject: [PATCH 83/96] fix unbound variable

---
 e2e/scripts/playwright.sh | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/e2e/scripts/playwright.sh b/e2e/scripts/playwright.sh
index 376c8d51d..8fc65e854 100755
--- a/e2e/scripts/playwright.sh
+++ b/e2e/scripts/playwright.sh
@@ -17,9 +17,7 @@ main() {
     export DR_AWS_SECRET_ACCESS_KEY="$7"
   elif [ "$test_name" == "deploy-upgrade" ]; then
     export APP_UPGRADE_VERSION="$2"
-    if [ -n "$3" ]; then
-      export SKIP_CLUSTER_UPGRADE_CHECK="$3"
-    fi
+    export SKIP_CLUSTER_UPGRADE_CHECK="${3:-}"
   fi
 
   export BASE_URL="http://10.0.0.2:30001"

From ff222cd159543f3931ff303c36716e2416ba54ce Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Fri, 12 Jul 2024 16:24:59 -0700
Subject: [PATCH 84/96] use ttl.sh kotsadm image

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index b8c073c3e..7f5b2318e 100644
--- a/Makefile
+++ b/Makefile
@@ -4,7 +4,7 @@ ARCH := $(shell uname -m)
 APP_NAME = embedded-cluster
 ADMIN_CONSOLE_CHART_REPO_OVERRIDE =
 ADMIN_CONSOLE_CHART_VERSION = 1.110.0
-ADMIN_CONSOLE_IMAGE_OVERRIDE = kotsadm/kotsadm:v2024.7.8-ec8-nightly
+ADMIN_CONSOLE_IMAGE_OVERRIDE = ttl.sh/sgalsaleh/kotsadm:24h
 ADMIN_CONSOLE_MIGRATIONS_IMAGE_OVERRIDE = kotsadm/kotsadm-migrations:v2024.7.8-ec8-nightly
 ADMIN_CONSOLE_KURL_PROXY_IMAGE_OVERRIDE = kotsadm/kurl-proxy:v2024.7.8-ec8-nightly
 EMBEDDED_OPERATOR_CHART_VERSION = 0.39.0-build.3

From c959f7cc29496cc79527f363333ba6e0307f607d Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Sat, 13 Jul 2024 10:49:02 -0700
Subject: [PATCH 85/96] override kots binary

---
 Makefile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index 7f5b2318e..7403a3efd 100644
--- a/Makefile
+++ b/Makefile
@@ -26,8 +26,8 @@ PREVIOUS_K0S_VERSION ?= v1.28.10+k0s.0
 K0S_BINARY_SOURCE_OVERRIDE = https://ec-k0s-binaries.s3.amazonaws.com/k0s-v1.29.5%2Bk0s.0-ec.0
 PREVIOUS_K0S_BINARY_SOURCE_OVERRIDE =
 TROUBLESHOOT_VERSION = v0.93.1
-KOTS_VERSION = v2024.7.8-ec8-nightly
-KOTS_BINARY_URL_OVERRIDE =
+KOTS_VERSION = v2024.7.13-ec1-nightly
+KOTS_BINARY_URL_OVERRIDE = https://c60a-172-58-211-103.ngrok-free.app/kots.tar.gz
 LOCAL_ARTIFACT_MIRROR_IMAGE ?= registry.replicated.com/library/embedded-cluster-local-artifact-mirror
 LOCAL_ARTIFACT_MIRROR_IMAGE_LOCATION = ${LOCAL_ARTIFACT_MIRROR_IMAGE}:$(subst +,-,$(VERSION))
 LD_FLAGS = -X github.com/replicatedhq/embedded-cluster/pkg/defaults.K0sVersion=$(K0S_VERSION) \

From e9b406d4b5fbdc8f38e8f8641147b36a2a952720 Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Sat, 13 Jul 2024 11:32:08 -0700
Subject: [PATCH 86/96] fix operator image and timeout

---
 Makefile                                         | 2 +-
 e2e/playwright/tests/deploy-upgrade/test.spec.ts | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index 7403a3efd..6adbc1d7d 100644
--- a/Makefile
+++ b/Makefile
@@ -10,7 +10,7 @@ ADMIN_CONSOLE_KURL_PROXY_IMAGE_OVERRIDE = kotsadm/kurl-proxy:v2024.7.8-ec8-night
 EMBEDDED_OPERATOR_CHART_VERSION = 0.39.0-build.3
 EMBEDDED_OPERATOR_BINARY_URL_OVERRIDE =
 EMBEDDED_OPERATOR_UTILS_IMAGE = busybox:1.36.1
-EMBEDDED_CLUSTER_OPERATOR_IMAGE_OVERRIDE = ttl.sh/embedded-cluster-operator-image:dev-a11a481
+EMBEDDED_CLUSTER_OPERATOR_IMAGE_OVERRIDE = ttl.sh/sgalsaleh/embedded-cluster-operator-image:24h
 OPENEBS_CHART_VERSION = 4.1.0
 OPENEBS_UTILS_VERSION = 4.1.0
 SEAWEEDFS_CHART_VERSION = 4.0.0
diff --git a/e2e/playwright/tests/deploy-upgrade/test.spec.ts b/e2e/playwright/tests/deploy-upgrade/test.spec.ts
index a860b15a0..94f990293 100644
--- a/e2e/playwright/tests/deploy-upgrade/test.spec.ts
+++ b/e2e/playwright/tests/deploy-upgrade/test.spec.ts
@@ -33,7 +33,8 @@ test('deploy upgrade', async ({ page }) => {
   }
 
   await expect(page.locator('.available-update-row', { hasText: process.env.APP_UPGRADE_VERSION })).not.toBeVisible({ timeout: 10 * 1000 });
-  await expect(page.locator('.VersionHistoryRow', { hasText: process.env.APP_UPGRADE_VERSION })).toContainText('Currently deployed version', { timeout: 30 * 1000 });
+  await expect(page.locator('.VersionHistoryRow', { hasText: process.env.APP_UPGRADE_VERSION })).toContainText('Deploying');
+  await expect(page.locator('.VersionHistoryRow', { hasText: process.env.APP_UPGRADE_VERSION })).toContainText('Currently deployed version', { timeout: 90 * 1000 });
   await page.getByRole('link', { name: 'Dashboard', exact: true }).click();
   await expect(page.locator('.VersionCard-content--wrapper')).toContainText(process.env.APP_UPGRADE_VERSION);
   await expect(page.locator('#app')).toContainText('Currently deployed version');

From 6803208884a46c165197105917adc7cd7a9adfc3 Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Sat, 13 Jul 2024 12:08:50 -0700
Subject: [PATCH 87/96] use correct locator

---
 e2e/playwright/tests/deploy-upgrade/test.spec.ts | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/e2e/playwright/tests/deploy-upgrade/test.spec.ts b/e2e/playwright/tests/deploy-upgrade/test.spec.ts
index 94f990293..70ca3316c 100644
--- a/e2e/playwright/tests/deploy-upgrade/test.spec.ts
+++ b/e2e/playwright/tests/deploy-upgrade/test.spec.ts
@@ -5,10 +5,7 @@ test('deploy upgrade', async ({ page }) => {
   test.setTimeout(15 * 60 * 1000); // 15 minutes
   await login(page);
   await page.getByRole('link', { name: 'Version history', exact: true }).click();
-  // TODO NOW: uncomment this
-  // await page.locator('.available-update-row', { hasText: process.env.APP_UPGRADE_VERSION }).getByRole('button', { name: 'Deploy', exact: true }).click();
-  // TODO NOW: remove this line
-  await page.getByText(process.env.APP_UPGRADE_VERSION, { exact: true }).locator('../../..').getByRole('button', { name: 'Deploy', exact: true }).click();
+  await page.locator('.available-update-row', { hasText: process.env.APP_UPGRADE_VERSION }).getByRole('button', { name: 'Deploy', exact: true }).click();
   const iframe = page.frameLocator('#upgrade-service-iframe');
   await expect(iframe.locator('h3')).toContainText('The First Config Group', { timeout: 20 * 1000 });
   await expect(iframe.locator('input[type="text"]')).toHaveValue('initial-hostname.com');
@@ -33,7 +30,6 @@ test('deploy upgrade', async ({ page }) => {
   }
 
   await expect(page.locator('.available-update-row', { hasText: process.env.APP_UPGRADE_VERSION })).not.toBeVisible({ timeout: 10 * 1000 });
-  await expect(page.locator('.VersionHistoryRow', { hasText: process.env.APP_UPGRADE_VERSION })).toContainText('Deploying');
   await expect(page.locator('.VersionHistoryRow', { hasText: process.env.APP_UPGRADE_VERSION })).toContainText('Currently deployed version', { timeout: 90 * 1000 });
   await page.getByRole('link', { name: 'Dashboard', exact: true }).click();
   await expect(page.locator('.VersionCard-content--wrapper')).toContainText(process.env.APP_UPGRADE_VERSION);

From 9e1285639bbbbca52bee67f5028bd29102b521f2 Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Tue, 16 Jul 2024 08:24:17 -0700
Subject: [PATCH 88/96] use official operator/kots releases

---
 Makefile | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/Makefile b/Makefile
index 6adbc1d7d..98f3abe81 100644
--- a/Makefile
+++ b/Makefile
@@ -3,14 +3,14 @@ UNAME := $(shell uname)
 ARCH := $(shell uname -m)
 APP_NAME = embedded-cluster
 ADMIN_CONSOLE_CHART_REPO_OVERRIDE =
-ADMIN_CONSOLE_CHART_VERSION = 1.110.0
-ADMIN_CONSOLE_IMAGE_OVERRIDE = ttl.sh/sgalsaleh/kotsadm:24h
-ADMIN_CONSOLE_MIGRATIONS_IMAGE_OVERRIDE = kotsadm/kotsadm-migrations:v2024.7.8-ec8-nightly
-ADMIN_CONSOLE_KURL_PROXY_IMAGE_OVERRIDE = kotsadm/kurl-proxy:v2024.7.8-ec8-nightly
-EMBEDDED_OPERATOR_CHART_VERSION = 0.39.0-build.3
+ADMIN_CONSOLE_CHART_VERSION = v1.112.0
+ADMIN_CONSOLE_IMAGE_OVERRIDE =
+ADMIN_CONSOLE_MIGRATIONS_IMAGE_OVERRIDE =
+ADMIN_CONSOLE_KURL_PROXY_IMAGE_OVERRIDE =
+EMBEDDED_OPERATOR_CHART_VERSION = 0.40.0
 EMBEDDED_OPERATOR_BINARY_URL_OVERRIDE =
 EMBEDDED_OPERATOR_UTILS_IMAGE = busybox:1.36.1
-EMBEDDED_CLUSTER_OPERATOR_IMAGE_OVERRIDE = ttl.sh/sgalsaleh/embedded-cluster-operator-image:24h
+EMBEDDED_CLUSTER_OPERATOR_IMAGE_OVERRIDE =
 OPENEBS_CHART_VERSION = 4.1.0
 OPENEBS_UTILS_VERSION = 4.1.0
 SEAWEEDFS_CHART_VERSION = 4.0.0
@@ -26,8 +26,8 @@ PREVIOUS_K0S_VERSION ?= v1.28.10+k0s.0
 K0S_BINARY_SOURCE_OVERRIDE = https://ec-k0s-binaries.s3.amazonaws.com/k0s-v1.29.5%2Bk0s.0-ec.0
 PREVIOUS_K0S_BINARY_SOURCE_OVERRIDE =
 TROUBLESHOOT_VERSION = v0.93.1
-KOTS_VERSION = v2024.7.13-ec1-nightly
-KOTS_BINARY_URL_OVERRIDE = https://c60a-172-58-211-103.ngrok-free.app/kots.tar.gz
+KOTS_VERSION =
+KOTS_BINARY_URL_OVERRIDE =
 LOCAL_ARTIFACT_MIRROR_IMAGE ?= registry.replicated.com/library/embedded-cluster-local-artifact-mirror
 LOCAL_ARTIFACT_MIRROR_IMAGE_LOCATION = ${LOCAL_ARTIFACT_MIRROR_IMAGE}:$(subst +,-,$(VERSION))
 LD_FLAGS = -X github.com/replicatedhq/embedded-cluster/pkg/defaults.K0sVersion=$(K0S_VERSION) \

From c3c7747578b56c1001cb892658bfb4ae3ed2183f Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Tue, 16 Jul 2024 08:26:09 -0700
Subject: [PATCH 89/96] fix Makefile

---
 Makefile | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index 98f3abe81..ec23898d7 100644
--- a/Makefile
+++ b/Makefile
@@ -7,8 +7,9 @@ ADMIN_CONSOLE_CHART_VERSION = v1.112.0
 ADMIN_CONSOLE_IMAGE_OVERRIDE =
 ADMIN_CONSOLE_MIGRATIONS_IMAGE_OVERRIDE =
 ADMIN_CONSOLE_KURL_PROXY_IMAGE_OVERRIDE =
+EMBEDDED_OPERATOR_CHART_URL = oci://registry.replicated.com/library
+EMBEDDED_OPERATOR_CHART_NAME = embedded-cluster-operator
 EMBEDDED_OPERATOR_CHART_VERSION = 0.40.0
-EMBEDDED_OPERATOR_BINARY_URL_OVERRIDE =
 EMBEDDED_OPERATOR_UTILS_IMAGE = busybox:1.36.1
 EMBEDDED_CLUSTER_OPERATOR_IMAGE_OVERRIDE =
 OPENEBS_CHART_VERSION = 4.1.0
@@ -26,7 +27,7 @@ PREVIOUS_K0S_VERSION ?= v1.28.10+k0s.0
 K0S_BINARY_SOURCE_OVERRIDE = https://ec-k0s-binaries.s3.amazonaws.com/k0s-v1.29.5%2Bk0s.0-ec.0
 PREVIOUS_K0S_BINARY_SOURCE_OVERRIDE =
 TROUBLESHOOT_VERSION = v0.93.1
-KOTS_VERSION =
+KOTS_VERSION = v$(shell echo $(ADMIN_CONSOLE_CHART_VERSION) | sed 's/\([0-9]\+\.[0-9]\+\.[0-9]\+\).*/\1/')
 KOTS_BINARY_URL_OVERRIDE =
 LOCAL_ARTIFACT_MIRROR_IMAGE ?= registry.replicated.com/library/embedded-cluster-local-artifact-mirror
 LOCAL_ARTIFACT_MIRROR_IMAGE_LOCATION = ${LOCAL_ARTIFACT_MIRROR_IMAGE}:$(subst +,-,$(VERSION))

From 28c7a33fa4f0e0398846f512b15da8f91742421c Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Tue, 16 Jul 2024 08:28:41 -0700
Subject: [PATCH 90/96] remove v

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index ec23898d7..6e488bf74 100644
--- a/Makefile
+++ b/Makefile
@@ -3,7 +3,7 @@ UNAME := $(shell uname)
 ARCH := $(shell uname -m)
 APP_NAME = embedded-cluster
 ADMIN_CONSOLE_CHART_REPO_OVERRIDE =
-ADMIN_CONSOLE_CHART_VERSION = v1.112.0
+ADMIN_CONSOLE_CHART_VERSION = 1.112.0
 ADMIN_CONSOLE_IMAGE_OVERRIDE =
 ADMIN_CONSOLE_MIGRATIONS_IMAGE_OVERRIDE =
 ADMIN_CONSOLE_KURL_PROXY_IMAGE_OVERRIDE =

From d98405c8aac93a28ee263e4f50ce6a0ca896bf7e Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Tue, 16 Jul 2024 08:42:47 -0700
Subject: [PATCH 91/96] print kots version

---
 scripts/cache-files.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/cache-files.sh b/scripts/cache-files.sh
index 109a6aaef..f44fb4abc 100755
--- a/scripts/cache-files.sh
+++ b/scripts/cache-files.sh
@@ -109,7 +109,7 @@ function operatorbin() {
 function kotsbin() {
     # first, figure out what version of kots is in the current build
     local kots_version=
-    kots_version=$(awk '/^KOTS_VERSION/{print $3}' Makefile)
+    kots_version=$(make print-KOTS_VERSION)
 
     local kots_override=
     kots_override=$(awk '/^KOTS_BINARY_URL_OVERRIDE/{gsub("\"", "", $3); print $3}' Makefile)

From ae232eab594e54102ef40daff1c79422708fa856 Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Tue, 16 Jul 2024 10:46:44 -0700
Subject: [PATCH 92/96] debug operator tarball

---
 scripts/cache-files.sh | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/scripts/cache-files.sh b/scripts/cache-files.sh
index f44fb4abc..2edba0dac 100755
--- a/scripts/cache-files.sh
+++ b/scripts/cache-files.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-set -euo pipefail
+set -euox pipefail
 
 function require() {
     if [ -z "$2" ]; then
@@ -97,10 +97,18 @@ function operatorbin() {
         curl --fail-with-body -L -o operator "https://github.com/replicatedhq/embedded-cluster-operator/releases/download/v${operator_version}/manager"
     fi
 
+    echo "downloaded operator binary, setting permissions"
+
     chmod +x operator
 
+    echo "compressing operator binary"
+
     # compress the operator binary
-    tar -czf "${operator_version}.tar.gz" operator
+    tar -czvf "${operator_version}.tar.gz" operator
+
+    # debug content of the tarball
+    tar -xzvf "${operator_version}.tar.gz"
+    ls -lah
 
     # upload the binary to the bucket
     retry 3 aws s3 cp --no-progress "${operator_version}.tar.gz" "s3://${S3_BUCKET}/operator-binaries/${operator_version}.tar.gz"

From 27952b4497ada2b62689339b3e33b254313af546 Mon Sep 17 00:00:00 2001
From: Salah Al Saleh <sg.alsaleh@gmail.com>
Date: Tue, 16 Jul 2024 18:12:00 +0000
Subject: [PATCH 93/96] operator 0.40.1

---
 Makefile | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/Makefile b/Makefile
index 6e488bf74..54fcd036d 100644
--- a/Makefile
+++ b/Makefile
@@ -7,9 +7,7 @@ ADMIN_CONSOLE_CHART_VERSION = 1.112.0
 ADMIN_CONSOLE_IMAGE_OVERRIDE =
 ADMIN_CONSOLE_MIGRATIONS_IMAGE_OVERRIDE =
 ADMIN_CONSOLE_KURL_PROXY_IMAGE_OVERRIDE =
-EMBEDDED_OPERATOR_CHART_URL = oci://registry.replicated.com/library
-EMBEDDED_OPERATOR_CHART_NAME = embedded-cluster-operator
-EMBEDDED_OPERATOR_CHART_VERSION = 0.40.0
+EMBEDDED_OPERATOR_CHART_VERSION = 0.40.1
 EMBEDDED_OPERATOR_UTILS_IMAGE = busybox:1.36.1
 EMBEDDED_CLUSTER_OPERATOR_IMAGE_OVERRIDE =
 OPENEBS_CHART_VERSION = 4.1.0

From babbfaff2e9923ee61ddc146cb0935ccb35d46aa Mon Sep 17 00:00:00 2001
From: Salah Al Saleh <sg.alsaleh@gmail.com>
Date: Tue, 16 Jul 2024 18:13:22 +0000
Subject: [PATCH 94/96] add EMBEDDED_OPERATOR_BINARY_URL_OVERRIDE =

---
 Makefile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Makefile b/Makefile
index 54fcd036d..27013c795 100644
--- a/Makefile
+++ b/Makefile
@@ -8,6 +8,7 @@ ADMIN_CONSOLE_IMAGE_OVERRIDE =
 ADMIN_CONSOLE_MIGRATIONS_IMAGE_OVERRIDE =
 ADMIN_CONSOLE_KURL_PROXY_IMAGE_OVERRIDE =
 EMBEDDED_OPERATOR_CHART_VERSION = 0.40.1
+EMBEDDED_OPERATOR_BINARY_URL_OVERRIDE =
 EMBEDDED_OPERATOR_UTILS_IMAGE = busybox:1.36.1
 EMBEDDED_CLUSTER_OPERATOR_IMAGE_OVERRIDE =
 OPENEBS_CHART_VERSION = 4.1.0

From ddc35a4fdfba52f5d9201b6557b51747dd4e5462 Mon Sep 17 00:00:00 2001
From: Salah Al Saleh <sg.alsaleh@gmail.com>
Date: Tue, 16 Jul 2024 19:21:21 +0000
Subject: [PATCH 95/96] operator 0.40.2

---
 Makefile               | 2 +-
 scripts/cache-files.sh | 8 --------
 2 files changed, 1 insertion(+), 9 deletions(-)

diff --git a/Makefile b/Makefile
index 27013c795..fd582fb7b 100644
--- a/Makefile
+++ b/Makefile
@@ -7,7 +7,7 @@ ADMIN_CONSOLE_CHART_VERSION = 1.112.0
 ADMIN_CONSOLE_IMAGE_OVERRIDE =
 ADMIN_CONSOLE_MIGRATIONS_IMAGE_OVERRIDE =
 ADMIN_CONSOLE_KURL_PROXY_IMAGE_OVERRIDE =
-EMBEDDED_OPERATOR_CHART_VERSION = 0.40.1
+EMBEDDED_OPERATOR_CHART_VERSION = 0.40.2
 EMBEDDED_OPERATOR_BINARY_URL_OVERRIDE =
 EMBEDDED_OPERATOR_UTILS_IMAGE = busybox:1.36.1
 EMBEDDED_CLUSTER_OPERATOR_IMAGE_OVERRIDE =
diff --git a/scripts/cache-files.sh b/scripts/cache-files.sh
index 2edba0dac..149f7a42c 100755
--- a/scripts/cache-files.sh
+++ b/scripts/cache-files.sh
@@ -97,19 +97,11 @@ function operatorbin() {
         curl --fail-with-body -L -o operator "https://github.com/replicatedhq/embedded-cluster-operator/releases/download/v${operator_version}/manager"
     fi
 
-    echo "downloaded operator binary, setting permissions"
-
     chmod +x operator
 
-    echo "compressing operator binary"
-
     # compress the operator binary
     tar -czvf "${operator_version}.tar.gz" operator
 
-    # debug content of the tarball
-    tar -xzvf "${operator_version}.tar.gz"
-    ls -lah
-
     # upload the binary to the bucket
     retry 3 aws s3 cp --no-progress "${operator_version}.tar.gz" "s3://${S3_BUCKET}/operator-binaries/${operator_version}.tar.gz"
 }

From 5a8edc1b20815ed64ad0df6718340473ae065883 Mon Sep 17 00:00:00 2001
From: Salah Al Saleh <sg.alsaleh@gmail.com>
Date: Tue, 16 Jul 2024 21:08:45 +0000
Subject: [PATCH 96/96] kots 1.112.1

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index fd582fb7b..45022f6fe 100644
--- a/Makefile
+++ b/Makefile
@@ -3,7 +3,7 @@ UNAME := $(shell uname)
 ARCH := $(shell uname -m)
 APP_NAME = embedded-cluster
 ADMIN_CONSOLE_CHART_REPO_OVERRIDE =
-ADMIN_CONSOLE_CHART_VERSION = 1.112.0
+ADMIN_CONSOLE_CHART_VERSION = 1.112.1
 ADMIN_CONSOLE_IMAGE_OVERRIDE =
 ADMIN_CONSOLE_MIGRATIONS_IMAGE_OVERRIDE =
 ADMIN_CONSOLE_KURL_PROXY_IMAGE_OVERRIDE =