install operator with no custom values in install2 (#1756)

* install operator with no custom values in install2

* check installation state is installed in install2 tests

* use CRD based installation type

* use correct CRD kind, apply multiple CRDs, wait for right condition

* recreate crd object for second loop

* add loading spinner, decrease wait times, remove debug logs

* allow crd to be taken over by helm

* remove prefix from installation name

* formatting

* set installation status after upgrade

* Revert "set installation status after upgrade"

This reverts commit 1b6cd62.

* Revert "remove prefix from installation name"

This reverts commit 226950b.

cmd/installer/cli/install2.go

@@ -5,12 +5,14 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"strings"
 	"time"
 
 	k0sconfig "github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1"
 	k0sv1beta1 "github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1"
 	"github.com/replicatedhq/embedded-cluster/cmd/installer/kotscli"
 	ecv1beta1 "github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1"
+	"github.com/replicatedhq/embedded-cluster/operator/charts"
 	"github.com/replicatedhq/embedded-cluster/pkg/addons2"
 	"github.com/replicatedhq/embedded-cluster/pkg/configutils"
 	"github.com/replicatedhq/embedded-cluster/pkg/extensions"
@@ -29,9 +31,12 @@ import (
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	corev1 "k8s.io/api/core/v1"
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/yaml"
 )
 
 type Install2CmdFlags struct {
@@ -478,15 +483,25 @@ func installAndStartCluster(ctx context.Context, networkInterface string, airgap
 }
 
 func recordInstallation(ctx context.Context, flags Install2CmdFlags, k0sCfg *k0sv1beta1.ClusterConfig, disasterRecoveryEnabled bool) (*ecv1beta1.Installation, error) {
+	loading := spinner.Start()
+	defer loading.Close()
+	loading.Infof("Creating types")
+
 	kcli, err := kubeutils.KubeClient()
 	if err != nil {
 		return nil, fmt.Errorf("create kube client: %w", err)
 	}
 
+	// ensure that the embedded-cluster namespace exists
 	if err := createECNamespace(ctx, kcli); err != nil {
 		return nil, fmt.Errorf("create embedded-cluster namespace: %w", err)
 	}
 
+	// ensure that the installation CRD exists
+	if err := createInstallationCRD(ctx, kcli); err != nil {
+		return nil, fmt.Errorf("create installation CRD: %w", err)
+	}
+
 	cfg, err := release.GetEmbeddedClusterConfig()
 	if err != nil {
 		return nil, err
@@ -525,7 +540,7 @@ func recordInstallation(ctx context.Context, flags Install2CmdFlags, k0sCfg *k0s
 			RuntimeConfig:             runtimeconfig.Get(),
 			EndUserK0sConfigOverrides: euOverrides,
 			BinaryName:                runtimeconfig.BinaryName(),
-			SourceType:                ecv1beta1.InstallationSourceTypeConfigMap,
+			SourceType:                ecv1beta1.InstallationSourceTypeCRD,
 			LicenseInfo: &ecv1beta1.LicenseInfo{
 				IsDisasterRecoverySupported: disasterRecoveryEnabled,
 			},
@@ -538,6 +553,11 @@ func recordInstallation(ctx context.Context, flags Install2CmdFlags, k0sCfg *k0s
 		return nil, fmt.Errorf("create installation: %w", err)
 	}
 
+	if err := kubeutils.UpdateInstallationStatus(ctx, kcli, &installation); err != nil {
+		return nil, fmt.Errorf("update installation status: %w", err)
+	}
+
+	loading.Infof("Types created!")
 	return &installation, nil
 }
 
@@ -547,7 +567,7 @@ func updateInstallation(ctx context.Context, install *ecv1beta1.Installation) er
 		return fmt.Errorf("create kube client: %w", err)
 	}
 
-	if err := kubeutils.UpdateInstallation(ctx, kcli, install); err != nil {
+	if err := kubeutils.UpdateInstallationStatus(ctx, kcli, install); err != nil {
 		return fmt.Errorf("update installation")
 	}
 	return nil
@@ -565,6 +585,54 @@ func createECNamespace(ctx context.Context, kcli client.Client) error {
 	return nil
 }
 
+func createInstallationCRD(ctx context.Context, kcli client.Client) error {
+	// decode the CRD file
+	crds := strings.Split(charts.InstallationCRDFile, "\n---\n")
+
+	for _, crdYaml := range crds {
+		var crd apiextensionsv1.CustomResourceDefinition
+		if err := yaml.Unmarshal([]byte(crdYaml), &crd); err != nil {
+			return fmt.Errorf("unmarshal installation CRD: %w", err)
+		}
+
+		// apply labels and annotations so that the CRD can be taken over by helm shortly
+		if crd.Labels == nil {
+			crd.Labels = map[string]string{}
+		}
+		crd.Labels["app.kubernetes.io/managed-by"] = "Helm"
+		if crd.Annotations == nil {
+			crd.Annotations = map[string]string{}
+		}
+		crd.Annotations["meta.helm.sh/release-name"] = "embedded-cluster-operator"
+		crd.Annotations["meta.helm.sh/release-namespace"] = "embedded-cluster"
+
+		// apply the CRD
+		if err := kcli.Create(ctx, &crd); err != nil {
+			return fmt.Errorf("apply installation CRD: %w", err)
+		}
+
+		// wait for the CRD to be ready
+		backoff := wait.Backoff{Steps: 600, Duration: 100 * time.Millisecond, Factor: 1.0, Jitter: 0.1}
+		if err := wait.ExponentialBackoffWithContext(ctx, backoff, func(ctx context.Context) (bool, error) {
+			newCrd := apiextensionsv1.CustomResourceDefinition{}
+			err := kcli.Get(ctx, client.ObjectKey{Name: crd.Name}, &newCrd)
+			if err != nil {
+				return false, nil // not ready yet
+			}
+			for _, cond := range newCrd.Status.Conditions {
+				if cond.Type == apiextensionsv1.Established && cond.Status == apiextensionsv1.ConditionTrue {
+					return true, nil
+				}
+			}
+			return false, nil
+		}); err != nil {
+			return fmt.Errorf("wait for installation CRD to be ready: %w", err)
+		}
+	}
+
+	return nil
+}
+
 func networkSpecFromK0sConfig(k0sCfg *k0sv1beta1.ClusterConfig) *ecv1beta1.NetworkSpec {
 	network := &ecv1beta1.NetworkSpec{}
 

e2e/scripts/check-installation-state2.sh

@@ -19,7 +19,15 @@ main() {
     echo "pods"
     kubectl get pods -A
 
-    echo "TODO: check installation configmap state"
+    echo "ensure that installation is installed"
+    if echo "$version" | grep "pre-minio-removal"; then
+        echo "waiting for installation as this is a pre-minio-removal embedded-cluster version (and so the installer doesn't wait for the installation to be ready itself)"
+        wait_for_installation
+    fi
+    if ! ensure_installation_is_installed; then
+        echo "installation is not installed"
+        exit 1
+    fi
 
     if ! wait_for_nginx_pods; then
         echo "Failed waiting for the application's nginx pods"

kinds/apis/v1beta1/installation_types.go

@@ -45,8 +45,7 @@ const (
 
 // Valid installation source types
 const (
-	InstallationSourceTypeCRD       string = "CRD"
-	InstallationSourceTypeConfigMap string = "ConfigMap"
+	InstallationSourceTypeCRD string = "CRD"
 )
 
 const (

operator/charts/crd.go

@@ -0,0 +1,8 @@
+package charts
+
+// this package is used to embed the installation CRD file into the binary
+
+import _ "embed"
+
+//go:embed embedded-cluster-operator/charts/crds/templates/resources.yaml
+var InstallationCRDFile string

pkg/addons2/addons.go

@@ -6,6 +6,7 @@ import (
 	"github.com/pkg/errors"
 	ecv1beta1 "github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1"
 	"github.com/replicatedhq/embedded-cluster/pkg/addons2/adminconsole"
+	"github.com/replicatedhq/embedded-cluster/pkg/addons2/embeddedclusteroperator"
 	"github.com/replicatedhq/embedded-cluster/pkg/addons2/openebs"
 	"github.com/replicatedhq/embedded-cluster/pkg/addons2/registry"
 	"github.com/replicatedhq/embedded-cluster/pkg/addons2/types"
@@ -69,6 +70,7 @@ func Install(ctx context.Context, opts InstallOptions) error {
 func getAddOns(opts InstallOptions) []types.AddOn {
 	addOns := []types.AddOn{
 		&openebs.OpenEBS{},
+		&embeddedclusteroperator.EmbeddedClusterOperator{},
 	}
 
 	if opts.AirgapBundle != "" {

pkg/addons2/embeddedclusteroperator/embeddedclusteroperator.go

@@ -0,0 +1,42 @@
+package embeddedclusteroperator
+
+import (
+	_ "embed"
+
+	"github.com/pkg/errors"
+	"github.com/replicatedhq/embedded-cluster/pkg/release"
+	"gopkg.in/yaml.v2"
+)
+
+type EmbeddedClusterOperator struct{}
+
+const (
+	releaseName = "embedded-cluster-operator"
+	namespace   = "embedded-cluster"
+)
+
+var (
+	//go:embed static/values.tpl.yaml
+	rawvalues []byte
+	// helmValues is the unmarshal version of rawvalues.
+	helmValues map[string]interface{}
+	//go:embed static/metadata.yaml
+	rawmetadata []byte
+	// Metadata is the unmarshal version of rawmetadata.
+	Metadata release.AddonMetadata
+)
+
+func init() {
+	if err := yaml.Unmarshal(rawmetadata, &Metadata); err != nil {
+		panic(errors.Wrap(err, "unable to unmarshal metadata"))
+	}
+	hv, err := release.RenderHelmValues(rawvalues, Metadata)
+	if err != nil {
+		panic(errors.Wrap(err, "unable to unmarshal values"))
+	}
+	helmValues = hv
+}
+
+func (a *EmbeddedClusterOperator) Name() string {
+	return "Embedded Cluster Operator"
+}

pkg/addons2/embeddedclusteroperator/install.go

@@ -0,0 +1,29 @@
+package embeddedclusteroperator
+
+import (
+	"context"
+
+	"github.com/pkg/errors"
+	"github.com/replicatedhq/embedded-cluster/pkg/helm"
+	"github.com/replicatedhq/embedded-cluster/pkg/spinner"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+func (o *EmbeddedClusterOperator) Install(ctx context.Context, kcli client.Client, hcli *helm.Helm, writer *spinner.MessageWriter) error {
+	if err := o.prepare(); err != nil {
+		return errors.Wrap(err, "prepare metrics operator")
+	}
+
+	_, err := hcli.Install(ctx, helm.InstallOptions{
+		ReleaseName:  releaseName,
+		ChartPath:    Metadata.Location,
+		ChartVersion: Metadata.Version,
+		Values:       helmValues,
+		Namespace:    namespace,
+	})
+	if err != nil {
+		return errors.Wrap(err, "install metrics operator")
+	}
+
+	return nil
+}

pkg/addons2/embeddedclusteroperator/prepare.go

@@ -0,0 +1,17 @@
+package embeddedclusteroperator
+
+import (
+	"github.com/pkg/errors"
+)
+
+func (a *EmbeddedClusterOperator) prepare() error {
+	if err := a.generateHelmValues(); err != nil {
+		return errors.Wrap(err, "generate helm values")
+	}
+
+	return nil
+}
+
+func (a *EmbeddedClusterOperator) generateHelmValues() error {
+	return nil
+}

pkg/addons2/embeddedclusteroperator/static/metadata.yaml

@@ -0,0 +1,20 @@
+#
+# this file is automatically generated by buildtools. manual edits are not recommended.
+# to regenerate this file, run the following commands:
+#
+# $ make buildtools
+# $ output/bin/buildtools update addon <addon name>
+#
+version: 1.19.0+k8s-1.30
+location: oci://proxy.replicated.com/anonymous/registry.replicated.com/library/embedded-cluster-operator
+images:
+    embedded-cluster-operator:
+        repo: proxy.replicated.com/anonymous/replicated/embedded-cluster-operator-image
+        tag:
+            amd64: v1.19.0-k8s-1.30
+            arm64: v1.19.0-k8s-1.30
+    utils:
+        repo: proxy.replicated.com/anonymous/replicated/ec-utils
+        tag:
+            amd64: latest
+            arm64: latest

pkg/addons2/embeddedclusteroperator/static/values.tpl.yaml

@@ -0,0 +1,10 @@
+global:
+  labels:
+    replicated.com/disaster-recovery: infra
+    replicated.com/disaster-recovery-chart: embedded-cluster-operator
+{{- if .ReplaceImages }}
+image:
+  repository: '{{ (index .Images "embedded-cluster-operator").Repo }}'
+  tag: '{{ index (index .Images "embedded-cluster-operator").Tag .GOARCH }}'
+utilsImage: '{{ ImageString (index .Images "utils") }}'
+{{- end }}