From 6070af7d75b7de0e6eb04898370e8dba1b0b95b6 Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Wed, 29 Jan 2025 11:23:06 -0500 Subject: [PATCH] install operator with no custom values in install2 (#1756) * install operator with no custom values in install2 * check installation state is installed in install2 tests * use CRD based installation type * use correct CRD kind, apply multiple CRDs, wait for right condition * recreate crd object for second loop * add loading spinner, decrease wait times, remove debug logs * allow crd to be taken over by helm * remove prefix from installation name * formatting * set installation status after upgrade * Revert "set installation status after upgrade" This reverts commit 1b6cd624f7f36331d0f235541f0f6679869a4f75. * Revert "remove prefix from installation name" This reverts commit 226950be5d287e11881a77794b4e3ad7a9e0977e. --- cmd/installer/cli/install2.go | 72 ++++++++++++++- e2e/scripts/check-installation-state2.sh | 10 +- kinds/apis/v1beta1/installation_types.go | 3 +- operator/charts/crd.go | 8 ++ pkg/addons2/addons.go | 2 + .../embeddedclusteroperator.go | 42 +++++++++ .../embeddedclusteroperator/install.go | 29 ++++++ .../embeddedclusteroperator/prepare.go | 17 ++++ .../static/metadata.yaml | 20 ++++ .../static/values.tpl.yaml | 10 ++ pkg/kubeutils/kubeutils.go | 91 ++----------------- pkg/release/addon.go | 8 +- 12 files changed, 217 insertions(+), 95 deletions(-) create mode 100644 operator/charts/crd.go create mode 100644 pkg/addons2/embeddedclusteroperator/embeddedclusteroperator.go create mode 100644 pkg/addons2/embeddedclusteroperator/install.go create mode 100644 pkg/addons2/embeddedclusteroperator/prepare.go create mode 100644 pkg/addons2/embeddedclusteroperator/static/metadata.yaml create mode 100644 pkg/addons2/embeddedclusteroperator/static/values.tpl.yaml diff --git a/cmd/installer/cli/install2.go b/cmd/installer/cli/install2.go index 127324db8..eafd81e29 100644 --- a/cmd/installer/cli/install2.go +++ b/cmd/installer/cli/install2.go @@ -5,12 +5,14 @@ import ( "errors" "fmt" "os" + "strings" "time" k0sconfig "github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1" k0sv1beta1 "github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1" "github.com/replicatedhq/embedded-cluster/cmd/installer/kotscli" ecv1beta1 "github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1" + "github.com/replicatedhq/embedded-cluster/operator/charts" "github.com/replicatedhq/embedded-cluster/pkg/addons2" "github.com/replicatedhq/embedded-cluster/pkg/configutils" "github.com/replicatedhq/embedded-cluster/pkg/extensions" @@ -29,9 +31,12 @@ import ( "github.com/sirupsen/logrus" "github.com/spf13/cobra" corev1 "k8s.io/api/core/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/yaml" ) type Install2CmdFlags struct { @@ -478,15 +483,25 @@ func installAndStartCluster(ctx context.Context, networkInterface string, airgap } func recordInstallation(ctx context.Context, flags Install2CmdFlags, k0sCfg *k0sv1beta1.ClusterConfig, disasterRecoveryEnabled bool) (*ecv1beta1.Installation, error) { + loading := spinner.Start() + defer loading.Close() + loading.Infof("Creating types") + kcli, err := kubeutils.KubeClient() if err != nil { return nil, fmt.Errorf("create kube client: %w", err) } + // ensure that the embedded-cluster namespace exists if err := createECNamespace(ctx, kcli); err != nil { return nil, fmt.Errorf("create embedded-cluster namespace: %w", err) } + // ensure that the installation CRD exists + if err := createInstallationCRD(ctx, kcli); err != nil { + return nil, fmt.Errorf("create installation CRD: %w", err) + } + cfg, err := release.GetEmbeddedClusterConfig() if err != nil { return nil, err @@ -525,7 +540,7 @@ func recordInstallation(ctx context.Context, flags Install2CmdFlags, k0sCfg *k0s RuntimeConfig: runtimeconfig.Get(), EndUserK0sConfigOverrides: euOverrides, BinaryName: runtimeconfig.BinaryName(), - SourceType: ecv1beta1.InstallationSourceTypeConfigMap, + SourceType: ecv1beta1.InstallationSourceTypeCRD, LicenseInfo: &ecv1beta1.LicenseInfo{ IsDisasterRecoverySupported: disasterRecoveryEnabled, }, @@ -538,6 +553,11 @@ func recordInstallation(ctx context.Context, flags Install2CmdFlags, k0sCfg *k0s return nil, fmt.Errorf("create installation: %w", err) } + if err := kubeutils.UpdateInstallationStatus(ctx, kcli, &installation); err != nil { + return nil, fmt.Errorf("update installation status: %w", err) + } + + loading.Infof("Types created!") return &installation, nil } @@ -547,7 +567,7 @@ func updateInstallation(ctx context.Context, install *ecv1beta1.Installation) er return fmt.Errorf("create kube client: %w", err) } - if err := kubeutils.UpdateInstallation(ctx, kcli, install); err != nil { + if err := kubeutils.UpdateInstallationStatus(ctx, kcli, install); err != nil { return fmt.Errorf("update installation") } return nil @@ -565,6 +585,54 @@ func createECNamespace(ctx context.Context, kcli client.Client) error { return nil } +func createInstallationCRD(ctx context.Context, kcli client.Client) error { + // decode the CRD file + crds := strings.Split(charts.InstallationCRDFile, "\n---\n") + + for _, crdYaml := range crds { + var crd apiextensionsv1.CustomResourceDefinition + if err := yaml.Unmarshal([]byte(crdYaml), &crd); err != nil { + return fmt.Errorf("unmarshal installation CRD: %w", err) + } + + // apply labels and annotations so that the CRD can be taken over by helm shortly + if crd.Labels == nil { + crd.Labels = map[string]string{} + } + crd.Labels["app.kubernetes.io/managed-by"] = "Helm" + if crd.Annotations == nil { + crd.Annotations = map[string]string{} + } + crd.Annotations["meta.helm.sh/release-name"] = "embedded-cluster-operator" + crd.Annotations["meta.helm.sh/release-namespace"] = "embedded-cluster" + + // apply the CRD + if err := kcli.Create(ctx, &crd); err != nil { + return fmt.Errorf("apply installation CRD: %w", err) + } + + // wait for the CRD to be ready + backoff := wait.Backoff{Steps: 600, Duration: 100 * time.Millisecond, Factor: 1.0, Jitter: 0.1} + if err := wait.ExponentialBackoffWithContext(ctx, backoff, func(ctx context.Context) (bool, error) { + newCrd := apiextensionsv1.CustomResourceDefinition{} + err := kcli.Get(ctx, client.ObjectKey{Name: crd.Name}, &newCrd) + if err != nil { + return false, nil // not ready yet + } + for _, cond := range newCrd.Status.Conditions { + if cond.Type == apiextensionsv1.Established && cond.Status == apiextensionsv1.ConditionTrue { + return true, nil + } + } + return false, nil + }); err != nil { + return fmt.Errorf("wait for installation CRD to be ready: %w", err) + } + } + + return nil +} + func networkSpecFromK0sConfig(k0sCfg *k0sv1beta1.ClusterConfig) *ecv1beta1.NetworkSpec { network := &ecv1beta1.NetworkSpec{} diff --git a/e2e/scripts/check-installation-state2.sh b/e2e/scripts/check-installation-state2.sh index 52b676675..b1a5f2609 100755 --- a/e2e/scripts/check-installation-state2.sh +++ b/e2e/scripts/check-installation-state2.sh @@ -19,7 +19,15 @@ main() { echo "pods" kubectl get pods -A - echo "TODO: check installation configmap state" + echo "ensure that installation is installed" + if echo "$version" | grep "pre-minio-removal"; then + echo "waiting for installation as this is a pre-minio-removal embedded-cluster version (and so the installer doesn't wait for the installation to be ready itself)" + wait_for_installation + fi + if ! ensure_installation_is_installed; then + echo "installation is not installed" + exit 1 + fi if ! wait_for_nginx_pods; then echo "Failed waiting for the application's nginx pods" diff --git a/kinds/apis/v1beta1/installation_types.go b/kinds/apis/v1beta1/installation_types.go index a7f2dcd6d..dfd6cc34b 100644 --- a/kinds/apis/v1beta1/installation_types.go +++ b/kinds/apis/v1beta1/installation_types.go @@ -45,8 +45,7 @@ const ( // Valid installation source types const ( - InstallationSourceTypeCRD string = "CRD" - InstallationSourceTypeConfigMap string = "ConfigMap" + InstallationSourceTypeCRD string = "CRD" ) const ( diff --git a/operator/charts/crd.go b/operator/charts/crd.go new file mode 100644 index 000000000..b0a58e2d9 --- /dev/null +++ b/operator/charts/crd.go @@ -0,0 +1,8 @@ +package charts + +// this package is used to embed the installation CRD file into the binary + +import _ "embed" + +//go:embed embedded-cluster-operator/charts/crds/templates/resources.yaml +var InstallationCRDFile string diff --git a/pkg/addons2/addons.go b/pkg/addons2/addons.go index 411f55462..bbf75d0d1 100644 --- a/pkg/addons2/addons.go +++ b/pkg/addons2/addons.go @@ -6,6 +6,7 @@ import ( "github.com/pkg/errors" ecv1beta1 "github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1" "github.com/replicatedhq/embedded-cluster/pkg/addons2/adminconsole" + "github.com/replicatedhq/embedded-cluster/pkg/addons2/embeddedclusteroperator" "github.com/replicatedhq/embedded-cluster/pkg/addons2/openebs" "github.com/replicatedhq/embedded-cluster/pkg/addons2/registry" "github.com/replicatedhq/embedded-cluster/pkg/addons2/types" @@ -69,6 +70,7 @@ func Install(ctx context.Context, opts InstallOptions) error { func getAddOns(opts InstallOptions) []types.AddOn { addOns := []types.AddOn{ &openebs.OpenEBS{}, + &embeddedclusteroperator.EmbeddedClusterOperator{}, } if opts.AirgapBundle != "" { diff --git a/pkg/addons2/embeddedclusteroperator/embeddedclusteroperator.go b/pkg/addons2/embeddedclusteroperator/embeddedclusteroperator.go new file mode 100644 index 000000000..1a3d3e636 --- /dev/null +++ b/pkg/addons2/embeddedclusteroperator/embeddedclusteroperator.go @@ -0,0 +1,42 @@ +package embeddedclusteroperator + +import ( + _ "embed" + + "github.com/pkg/errors" + "github.com/replicatedhq/embedded-cluster/pkg/release" + "gopkg.in/yaml.v2" +) + +type EmbeddedClusterOperator struct{} + +const ( + releaseName = "embedded-cluster-operator" + namespace = "embedded-cluster" +) + +var ( + //go:embed static/values.tpl.yaml + rawvalues []byte + // helmValues is the unmarshal version of rawvalues. + helmValues map[string]interface{} + //go:embed static/metadata.yaml + rawmetadata []byte + // Metadata is the unmarshal version of rawmetadata. + Metadata release.AddonMetadata +) + +func init() { + if err := yaml.Unmarshal(rawmetadata, &Metadata); err != nil { + panic(errors.Wrap(err, "unable to unmarshal metadata")) + } + hv, err := release.RenderHelmValues(rawvalues, Metadata) + if err != nil { + panic(errors.Wrap(err, "unable to unmarshal values")) + } + helmValues = hv +} + +func (a *EmbeddedClusterOperator) Name() string { + return "Embedded Cluster Operator" +} diff --git a/pkg/addons2/embeddedclusteroperator/install.go b/pkg/addons2/embeddedclusteroperator/install.go new file mode 100644 index 000000000..a5a542dd2 --- /dev/null +++ b/pkg/addons2/embeddedclusteroperator/install.go @@ -0,0 +1,29 @@ +package embeddedclusteroperator + +import ( + "context" + + "github.com/pkg/errors" + "github.com/replicatedhq/embedded-cluster/pkg/helm" + "github.com/replicatedhq/embedded-cluster/pkg/spinner" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func (o *EmbeddedClusterOperator) Install(ctx context.Context, kcli client.Client, hcli *helm.Helm, writer *spinner.MessageWriter) error { + if err := o.prepare(); err != nil { + return errors.Wrap(err, "prepare metrics operator") + } + + _, err := hcli.Install(ctx, helm.InstallOptions{ + ReleaseName: releaseName, + ChartPath: Metadata.Location, + ChartVersion: Metadata.Version, + Values: helmValues, + Namespace: namespace, + }) + if err != nil { + return errors.Wrap(err, "install metrics operator") + } + + return nil +} diff --git a/pkg/addons2/embeddedclusteroperator/prepare.go b/pkg/addons2/embeddedclusteroperator/prepare.go new file mode 100644 index 000000000..ccaf7ce39 --- /dev/null +++ b/pkg/addons2/embeddedclusteroperator/prepare.go @@ -0,0 +1,17 @@ +package embeddedclusteroperator + +import ( + "github.com/pkg/errors" +) + +func (a *EmbeddedClusterOperator) prepare() error { + if err := a.generateHelmValues(); err != nil { + return errors.Wrap(err, "generate helm values") + } + + return nil +} + +func (a *EmbeddedClusterOperator) generateHelmValues() error { + return nil +} diff --git a/pkg/addons2/embeddedclusteroperator/static/metadata.yaml b/pkg/addons2/embeddedclusteroperator/static/metadata.yaml new file mode 100644 index 000000000..c09f2d735 --- /dev/null +++ b/pkg/addons2/embeddedclusteroperator/static/metadata.yaml @@ -0,0 +1,20 @@ +# +# this file is automatically generated by buildtools. manual edits are not recommended. +# to regenerate this file, run the following commands: +# +# $ make buildtools +# $ output/bin/buildtools update addon +# +version: 1.19.0+k8s-1.30 +location: oci://proxy.replicated.com/anonymous/registry.replicated.com/library/embedded-cluster-operator +images: + embedded-cluster-operator: + repo: proxy.replicated.com/anonymous/replicated/embedded-cluster-operator-image + tag: + amd64: v1.19.0-k8s-1.30 + arm64: v1.19.0-k8s-1.30 + utils: + repo: proxy.replicated.com/anonymous/replicated/ec-utils + tag: + amd64: latest + arm64: latest diff --git a/pkg/addons2/embeddedclusteroperator/static/values.tpl.yaml b/pkg/addons2/embeddedclusteroperator/static/values.tpl.yaml new file mode 100644 index 000000000..a2478e996 --- /dev/null +++ b/pkg/addons2/embeddedclusteroperator/static/values.tpl.yaml @@ -0,0 +1,10 @@ +global: + labels: + replicated.com/disaster-recovery: infra + replicated.com/disaster-recovery-chart: embedded-cluster-operator +{{- if .ReplaceImages }} +image: + repository: '{{ (index .Images "embedded-cluster-operator").Repo }}' + tag: '{{ index (index .Images "embedded-cluster-operator").Tag .GOARCH }}' +utilsImage: '{{ ImageString (index .Images "utils") }}' +{{- end }} diff --git a/pkg/kubeutils/kubeutils.go b/pkg/kubeutils/kubeutils.go index 8ae14e5a2..0eecda03e 100644 --- a/pkg/kubeutils/kubeutils.go +++ b/pkg/kubeutils/kubeutils.go @@ -2,7 +2,6 @@ package kubeutils import ( "context" - "encoding/json" "errors" "fmt" "io" @@ -228,34 +227,10 @@ func (k *KubeUtils) WaitForInstallation(ctx context.Context, cli client.Client, } func CreateInstallation(ctx context.Context, cli client.Client, in *ecv1beta1.Installation) error { - in.Spec.SourceType = ecv1beta1.InstallationSourceTypeConfigMap - - data, err := json.Marshal(in) - if err != nil { - return fmt.Errorf("marshal installation: %w", err) + in.Spec.SourceType = ecv1beta1.InstallationSourceTypeCRD + if err := cli.Create(ctx, in); err != nil { + return fmt.Errorf("create installation: %w", err) } - - cm := &corev1.ConfigMap{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "ConfigMap", - }, - ObjectMeta: metav1.ObjectMeta{ - Namespace: "embedded-cluster", - Name: in.Name, - Labels: map[string]string{ - "replicated.com/installation": "embedded-cluster", - "replicated.com/disaster-recovery": "ec-install", - }, - }, - Data: map[string]string{ - "installation": string(data), - }, - } - if err := cli.Create(ctx, cm); err != nil { - return fmt.Errorf("create configmap: %w", err) - } - return nil } @@ -267,24 +242,7 @@ func UpdateInstallation(ctx context.Context, cli client.Client, in *ecv1beta1.In } return nil } - - // find configmap with the same name as the installation - var cm corev1.ConfigMap - if err := cli.Get(ctx, types.NamespacedName{Namespace: "embedded-cluster", Name: in.Name}, &cm); err != nil { - return fmt.Errorf("get configmap: %w", err) - } - - // marshal the installation and update the configmap - data, err := json.Marshal(in) - if err != nil { - return fmt.Errorf("marshal installation: %w", err) - } - cm.Data["installation"] = string(data) - - if err := cli.Update(ctx, &cm); err != nil { - return fmt.Errorf("update configmap: %w", err) - } - return nil + return fmt.Errorf("update installation: source type %q not supported", in.Spec.SourceType) } func UpdateInstallationStatus(ctx context.Context, cli client.Client, in *ecv1beta1.Installation) error { @@ -296,36 +254,7 @@ func UpdateInstallationStatus(ctx context.Context, cli client.Client, in *ecv1be return nil } - return UpdateInstallation(ctx, cli, in) -} - -func ListCMInstallations(ctx context.Context, cli client.Client) ([]ecv1beta1.Installation, error) { - var cmList corev1.ConfigMapList - if err := cli.List(ctx, &cmList, - client.InNamespace("embedded-cluster"), - client.MatchingLabels(labels.Set{"replicated.com/installation": "embedded-cluster"}), - ); err != nil { - return nil, fmt.Errorf("list configmaps: %w", err) - } - - installs := []ecv1beta1.Installation{} - for _, cm := range cmList.Items { - var install ecv1beta1.Installation - data, ok := cm.Data["installation"] - if !ok { - return nil, fmt.Errorf("installation data not found in configmap %s/%s", cm.Namespace, cm.Name) - } - if err := json.Unmarshal([]byte(data), &install); err != nil { - return nil, fmt.Errorf("unmarshal installation: %w", err) - } - installs = append(installs, install) - } - - sort.SliceStable(installs, func(i, j int) bool { - return installs[j].Name < installs[i].Name - }) - - return installs, nil + return fmt.Errorf("update installation status: source type %q not supported", in.Spec.SourceType) } func ListCRDInstallations(ctx context.Context, cli client.Client) ([]ecv1beta1.Installation, error) { @@ -362,16 +291,8 @@ func ListCRDInstallations(ctx context.Context, cli client.Client) ([]ecv1beta1.I } func ListInstallations(ctx context.Context, cli client.Client) ([]ecv1beta1.Installation, error) { - installs, err := ListCMInstallations(ctx, cli) - if err != nil { - return nil, fmt.Errorf("list cm installations: %w", err) - } - if len(installs) > 0 { - return installs, nil - } - // fall back to CRD-based installations - installs, err = ListCRDInstallations(ctx, cli) + installs, err := ListCRDInstallations(ctx, cli) if err != nil { return nil, err } diff --git a/pkg/release/addon.go b/pkg/release/addon.go index 2d17c3356..0be3f47e3 100644 --- a/pkg/release/addon.go +++ b/pkg/release/addon.go @@ -115,11 +115,9 @@ func (a *AddonMetadata) Save(addon string) error { if err := os.WriteFile(fpath, buf.Bytes(), 0600); err != nil { return fmt.Errorf("failed to write addon metadata: %w", err) } - if addon != "embeddedclusteroperator" { - fpath = filepath.Join("pkg", "addons2", addon, "static", "metadata.yaml") - if err := os.WriteFile(fpath, buf.Bytes(), 0600); err != nil { - return fmt.Errorf("failed to write addon metadata: %w", err) - } + fpath = filepath.Join("pkg", "addons2", addon, "static", "metadata.yaml") + if err := os.WriteFile(fpath, buf.Bytes(), 0600); err != nil { + return fmt.Errorf("failed to write addon metadata: %w", err) } return nil }