Check in initial e2e tests for cloud operator. (#447)

Add an e2e test that installs the operator, then a crdbcluster, and
makes assertions about basic crdb functionality.

.github/workflows/ci.yaml

@@ -2,8 +2,7 @@ name: Helm Chart Package CI
 on:
   pull_request:
     branches:
-      - 'master'
-      - 'cert-manager-feature-branch'
+      - '*'
 
 jobs:
 

Makefile

@@ -112,6 +112,7 @@ test/cluster/down: bin/k3d
 
 test/e2e/%: PKG=$*
 test/e2e/%: bin/cockroach bin/kubectl bin/helm build/self-signer test/publish-images-to-k3d ## run e2e tests for package (e.g. install or rotate)
+	@bin/kubectl get node -o yaml
 	@PATH="$(PWD)/bin:${PATH}" go test -timeout 30m -v ./tests/e2e/$(PKG)/...
 
 test/lint: bin/helm ## lint the helm chart

go.mod

@@ -7,7 +7,7 @@ require (
 	github.com/cenkalti/backoff v2.2.1+incompatible
 	github.com/cockroachdb/cockroach-operator v0.0.0-20230531051823-2cb3e2e676f4
 	github.com/google/martian v2.1.1-0.20190517191504-25dcb96d9e51+incompatible
-	github.com/gruntwork-io/terratest v0.41.19
+	github.com/gruntwork-io/terratest v0.41.26
 	github.com/mitchellh/hashstructure/v2 v2.0.2
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.51.2

go.sum

@@ -341,8 +341,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t
 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
 github.com/gruntwork-io/go-commons v0.8.0 h1:k/yypwrPqSeYHevLlEDmvmgQzcyTwrlZGRaxEM6G0ro=
 github.com/gruntwork-io/go-commons v0.8.0/go.mod h1:gtp0yTtIBExIZp7vyIV9I0XQkVwiQZze678hvDXof78=
-github.com/gruntwork-io/terratest v0.41.19 h1:SCqYF28nHZuBlX+jZ+QI4bK+OLpPR8BgcxVUQxyQSEw=
-github.com/gruntwork-io/terratest v0.41.19/go.mod h1:O6gajNBjO1wvc7Wl9WtbO+ORcdnhAV2GQiBE71ycwIk=
+github.com/gruntwork-io/terratest v0.41.26 h1:ttDXBBDBAYV4KgP1itGQ5O61F6KwgMMUFHy64bzvuYU=
+github.com/gruntwork-io/terratest v0.41.26/go.mod h1:O6gajNBjO1wvc7Wl9WtbO+ORcdnhAV2GQiBE71ycwIk=
 github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
 github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
 github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=

operator/templates/_operator_certs.tpl

@@ -10,7 +10,7 @@
 {{- define "operator.certs" -}}
 {{- $days := default .Values.certificate.validForDays 3650 | int -}}
 {{- $ca := genCA "cockroach-operator-certs" 3650 -}}
-{{- $cert := genSignedCert "cert" nil (list "cockroach-webhook-service.default.svc" "cockroach-operator.default.svc") $days $ca -}}
+{{- $cert := genSignedCert "cert" nil (list (printf "cockroach-webhook-service.%s.svc" .Release.Namespace) (printf "cockroach-operator.%s.svc" .Release.Namespace)) $days $ca -}}
 ca.crt: {{ $ca.Cert | b64enc }}
 tls.crt: {{ $cert.Cert | b64enc }}
 tls.key: {{ $cert.Key | b64enc }}

operator/templates/cockroachdb-operator-certs.yaml

@@ -3,9 +3,9 @@ apiVersion: v1
 kind: Secret
 metadata:
   name: cockroach-operator-certs
-  namespace: default   # Change the namespace if needed
+  namespace: {{ .Release.Namespace }}
 type: Opaque
 data:
   {{ index $operatorCerts 0 }}
   {{ index $operatorCerts 1 }}
-  {{ index $operatorCerts 2 }}
+  {{ index $operatorCerts 2 }}

operator/templates/operator.yaml

@@ -380,7 +380,7 @@ apiVersion: v1
 kind: ServiceAccount
 metadata:
   name: cockroach-operator-default
-  namespace: default
+  namespace: {{ .Release.Namespace }}
   labels:
     app: cockroach-operator
 
@@ -397,7 +397,7 @@ roleRef:
   name: cockroach-operator-role
 subjects:
   - name: cockroach-operator-default
-    namespace: default
+    namespace: {{ .Release.Namespace }}
     kind: ServiceAccount
 
 ---
@@ -406,7 +406,7 @@ apiVersion: v1
 kind: Service
 metadata:
   name: cockroach-operator
-  namespace: default
+  namespace: {{ .Release.Namespace }}
   labels:
     app: cockroach-operator
 spec:
@@ -424,7 +424,7 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: cockroach-operator
-  namespace: default
+  namespace: {{ .Release.Namespace }}
   labels:
     app: cockroach-operator
 spec:
@@ -447,7 +447,7 @@ spec:
       priorityClassName: cockroach-operator
       containers:
         - name: cockroach-operator
-          image: {{ .Values.image.registry }}/{{ .Values.image.repository }}:release-2024-07-10-0-384-g53408b608c
+          image: {{ .Values.image.registry }}/{{ .Values.image.repository }}@sha256:af1ad8c772bb38a5702db1cb4b04728f096bccbad09ccf785ebb04007de8f336
           args:
             # Pin metrics port so it can be properly exposed by the "ports"
             # field below even in the event of a change to the default value.
@@ -461,7 +461,7 @@ spec:
             - name: grpc
               containerPort: 9070
           env:
-            - name: WATCH_NAMESPACE
+            - name: NAMESPACE
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.namespace
@@ -512,17 +512,6 @@ spec:
             periodSeconds: 3
             timeoutSeconds: 3
             failureThreshold: 3
-      # TODO(CC-27018) Use a node label that defines which nodes operator pods
-      # can run on, instead of blocking operator pods from running on some nodes.
-      affinity:
-        nodeAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-            nodeSelectorTerms:
-              - matchExpressions:
-                  - key: crdb.io/app
-                    operator: NotIn
-                    values:
-                      - "in-situ"
       volumes:
         - name: certs
           secret:

tests/e2e/install/cockroachdb_helm_e2e_test.go

@@ -1,6 +1,7 @@
 package integration
 
 import (
+	"encoding/json"
 	"fmt"
 	"io/fs"
 	"log"
@@ -25,16 +26,127 @@ import (
 	"github.com/cockroachdb/helm-charts/pkg/security"
 	util "github.com/cockroachdb/helm-charts/pkg/utils"
 	"github.com/cockroachdb/helm-charts/tests/testutil"
+	"github.com/gruntwork-io/terratest/modules/retry"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 var (
-	cfg              = ctrl.GetConfigOrDie()
-	k8sClient, _     = client.New(cfg, client.Options{})
-	releaseName      = "crdb-test"
-	customCASecret   = "custom-ca-secret"
-	helmChartPath, _ = filepath.Abs("../../../cockroachdb")
+	cfg                  = ctrl.GetConfigOrDie()
+	k8sClient, _         = client.New(cfg, client.Options{})
+	releaseName          = "crdb-test"
+	operatorReleaseName  = "crdb-operator-test"
+	customCASecret       = "custom-ca-secret"
+	helmChartPath, _     = filepath.Abs("../../../cockroachdb")
+	operatorChartPath, _ = filepath.Abs("../../../operator")
+	skipCleanup          = os.Getenv("SKIP_CLEANUP") != ""
 )
 
+func mustMarshalJson(value interface{}) string {
+	out, err := json.Marshal(value)
+	if err != nil {
+		panic(err)
+	}
+	return string(out)
+}
+
+func TestCockroachDBOperator(t *testing.T) {
+	namespaceName := fmt.Sprintf("cockroach-%s", strings.ToLower(t.Name()))
+	kubectlOptions := k8s.NewKubectlOptions("", "", namespaceName)
+
+	k8s.CreateNamespace(t, kubectlOptions, namespaceName)
+	if !skipCleanup {
+		defer k8s.DeleteNamespace(t, kubectlOptions, namespaceName)
+	}
+
+	const testDBName = "testdb"
+
+	extraArgs := map[string][]string{
+		"install": {
+			"--wait",
+			"--debug",
+		},
+	}
+
+	crdbCluster := testutil.CockroachCluster{
+		Cfg:              cfg,
+		K8sClient:        k8sClient,
+		StatefulSetName:  fmt.Sprintf("%s-cockroachdb", releaseName),
+		Namespace:        namespaceName,
+		ClientSecret:     fmt.Sprintf("%s-cockroachdb-client-secret", releaseName),
+		NodeSecret:       fmt.Sprintf("%s-cockroachdb-node-secret", releaseName),
+		CaSecret:         fmt.Sprintf("%s-cockroachdb-ca-secret", releaseName),
+		IsCaUserProvided: false,
+		DesiredNodes:     1,
+	}
+
+	// Deploy operator
+	operatorOpts := &helm.Options{
+		KubectlOptions: kubectlOptions,
+		ExtraArgs:      extraArgs,
+	}
+	helm.Install(t, operatorOpts, operatorChartPath, operatorReleaseName)
+	if !skipCleanup {
+		defer cleanupResources(
+			t,
+			operatorReleaseName,
+			kubectlOptions,
+			operatorOpts,
+			[]string{},
+		)
+	}
+
+	// Wait for crd to be installed
+	k8s.WaitUntilServiceAvailable(t, kubectlOptions, "cockroach-operator", 30, 2*time.Second)
+	retry.DoWithRetryE(t, "wait-for-crd", 60, time.Second*5, func() (string, error) {
+		return k8s.RunKubectlAndGetOutputE(t, operatorOpts.KubectlOptions, "get", "crd", "crdbclusters.crdb.cockroachlabs.com")
+	})
+
+	// Deploy crdb
+	crdbOpts := &helm.Options{
+		KubectlOptions: kubectlOptions,
+		SetValues: patchHelmValues(map[string]string{
+			"operator.enabled": "true",
+			"operator.dataStore.volumeClaimTemplate.spec.resources.requests.storage": "1Gi",
+		}),
+		SetJsonValues: map[string]string{
+			"operator.regions": mustMarshalJson([]map[string]interface{}{
+				{
+					"code":          "us-east-1",
+					"cloudProvider": "k3d",
+					"nodes":         crdbCluster.DesiredNodes,
+					"namespace":     namespaceName,
+				},
+			}),
+		},
+		ExtraArgs: extraArgs,
+	}
+	helm.Install(t, crdbOpts, helmChartPath, releaseName)
+	if !skipCleanup {
+		defer cleanupResources(
+			t,
+			releaseName,
+			kubectlOptions,
+			crdbOpts,
+			[]string{},
+		)
+	}
+
+	serviceName := fmt.Sprintf("%s-cockroachdb-public", releaseName)
+	k8s.WaitUntilServiceAvailable(t, kubectlOptions, serviceName, 30, 2*time.Second)
+
+	testutil.RequireCertificatesToBeValid(t, crdbCluster)
+	testutil.RequireCRDBClusterToBeReadyTimeout(t, kubectlOptions, crdbCluster, 600*time.Second)
+
+	pods := k8s.ListPods(t, kubectlOptions, metav1.ListOptions{
+		LabelSelector: "app=cockroachdb",
+	})
+	require.True(t, len(pods) > 0)
+	podName := fmt.Sprintf("%s.%s-cockroachdb", pods[0].Name, releaseName)
+
+	testutil.RequireCRDBClusterToFunction(t, crdbCluster, false, podName)
+	testutil.RequireCRDBDatabaseToFunction(t, crdbCluster, testDBName, podName)
+}
+
 func TestCockroachDbHelmInstall(t *testing.T) {
 	namespaceName := "cockroach" + strings.ToLower(random.UniqueId())
 	kubectlOptions := k8s.NewKubectlOptions("", "", namespaceName)
@@ -212,8 +324,10 @@ func TestCockroachDbHelmMigration(t *testing.T) {
 
 	cmdCa := shell.Command{
 		Command: "cockroach",
-		Args: []string{"cert", "create-ca", fmt.Sprintf("--certs-dir=%s", certsDir),
-			fmt.Sprintf("--ca-key=%s/ca.key", certsDir)},
+		Args: []string{
+			"cert", "create-ca", fmt.Sprintf("--certs-dir=%s", certsDir),
+			fmt.Sprintf("--ca-key=%s/ca.key", certsDir),
+		},
 		WorkingDir: ".",
 		Env:        nil,
 		Logger:     nil,
@@ -243,8 +357,10 @@ func TestCockroachDbHelmMigration(t *testing.T) {
 
 	cmdClient := shell.Command{
 		Command: "cockroach",
-		Args: []string{"cert", "create-client", security.RootUser, fmt.Sprintf("--certs-dir=%s", certsDir),
-			fmt.Sprintf("--ca-key=%s/ca.key", certsDir)},
+		Args: []string{
+			"cert", "create-client", security.RootUser, fmt.Sprintf("--certs-dir=%s", certsDir),
+			fmt.Sprintf("--ca-key=%s/ca.key", certsDir),
+		},
 		WorkingDir: ".",
 		Env:        nil,
 		Logger:     nil,
@@ -313,7 +429,7 @@ func TestCockroachDbHelmMigration(t *testing.T) {
 			"statefulset.updateStrategy.type": "OnDelete",
 		}),
 		ExtraArgs: map[string][]string{
-			"upgrade": []string{
+			"upgrade": {
 				"--timeout=20m",
 			},
 		},