Enable datadog chart EKS E2E tests in CI

.gitlab-ci.yml

@@ -1,7 +1,3 @@
-variables:
-  # Do not modify this - must be the repository name for Kubernetes gitlab runners to run
-  KUBERNETES_SERVICE_ACCOUNT_OVERWRITE: helm-charts
-
 stages:
   - e2e
 
@@ -14,22 +10,26 @@ e2e:
           - charts/**/*.yaml
           - test/**/*
         compare_to: "refs/heads/main"
-  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/test-infra-definitions/runner:986d192b
+  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/test-infra-definitions/runner:95dca87f269a
   tags: ["arch:amd64"]
+  variables:
+    AWS_KEYPAIR_NAME: datadog-agent-ci
+    AWS_PRIVATE_KEY_FILE: $CI_PROJECT_DIR/ssh_key
   before_script:
-    # Update awscli v2
-    - aws --version
-    - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
-    - unzip awscliv2.zip
-    - ./aws/install --update
-    - aws --version
-    # Setup AWS Credentials
     - mkdir -p ~/.aws
     - set +x
+    # Set GITHUB_TOKEN to avoid getting rate-limited when pulumi sdk downloads the kubernetes provider
+    - export GITHUB_TOKEN=$(aws ssm get-parameter --region us-east-1 --name ci.helm-charts.github_token --with-decryption --query "Parameter.Value" --out text)
+    # Configure AWS EC2 ssh key needed for create pulumi EKS environment
+    - aws ssm get-parameter --region us-east-1 --name ci.helm-charts.ssh_key --with-decryption --query "Parameter.Value" --out text > $AWS_PRIVATE_KEY_FILE
+    - set -x
+    # Without the newline ssh silently fails and moves on to try other auth methods
+    - echo "" >> $AWS_PRIVATE_KEY_FILE
+    - chmod 600 $AWS_PRIVATE_KEY_FILE
+    # Configure AWS profile
     - aws ssm get-parameter --region us-east-1 --name ci.helm-charts.e2e-agent-qa-profile --with-decryption --query "Parameter.Value" --out text >> ~/.aws/config
     - set -x
     - export AWS_PROFILE=agent-qa-ci
     - pulumi login "s3://dd-pulumi-state?region=us-east-1&awssdk=v2&profile=$AWS_PROFILE"
   script:
-    # TODO: enable running E2E tests when upstream pulumi-eks fix is merged https://github.com/pulumi/pulumi-eks/pull/886
-    - echo "Success!"
+    - make test-e2e

Makefile

@@ -46,15 +46,15 @@ vet:
 
 .PHONY: unit-test
 unit-test:
-	go test -C test ./... -count=1 -skip=E2E
+	go test -C test ./... -count=1
 
 .PHONY: update-test-baselines
 update-test-baselines:
-	go test -C test ./... -count=1 -skip=E2E -args -updateBaselines=true
+	go test -C test ./... -count=1 -args -updateBaselines=true
 
 .PHONY: integration-test
 integration-test:
-	go test -C test/integ --tags=integration -skip=E2E -count=1 -v
+	go test -C test/integ --tags=integration -count=1 -v
 
 # Running E2E tests locally:
 ## Must be connected to appgate
@@ -67,14 +67,14 @@ test-e2e: fmt vet e2e-test
 # aws-vault exec sso-agent-sandbox-account-admin -- make e2e-test
 .PHONY: e2e-test
 e2e-test:
-	E2E_CONFIG_PARAMS=$(E2E_CONFIG_PARAMS) E2E_PROFILE=$(E2E_PROFILE) go test -C test ./... -v -run=E2E -vet=off -timeout 1h -count=1
+	E2E_CONFIG_PARAMS=$(E2E_CONFIG_PARAMS) E2E_PROFILE=$(E2E_PROFILE) go test -C test/e2e ./... --tags=e2e -v -vet=off -timeout 1h -count=1
 
 # aws-vault exec sso-agent-sandbox-account-admin -- make e2e-test-preserve-stacks
 .PHONY: e2e-test-preserve-stacks
 e2e-test-preserve-stacks:
-	E2E_CONFIG_PARAMS=$(E2E_CONFIG_PARAMS) E2E_PROFILE=$(E2E_PROFILE) go test -C test ./... -v -run=E2E -vet=off -timeout 1h -count=1 -args -preserveStacks=true
+	E2E_CONFIG_PARAMS=$(E2E_CONFIG_PARAMS) E2E_PROFILE=$(E2E_PROFILE) go test -C test/e2e ./... --tags=e2e -v -vet=off -timeout 1h -count=1 -args -preserveStacks=true
 
 # aws-vault exec sso-agent-sandbox-account-admin -- make e2e-test-cleanup-stacks
 .PHONY: e2e-test-cleanup-stacks
 e2e-test-cleanup-stacks:
-	E2E_CONFIG_PARAMS=$(E2E_CONFIG_PARAMS) E2E_PROFILE=$(E2E_PROFILE) go test -C test ./... -v -run=E2E -vet=off -timeout 1h -count=1 -args -destroyStacks=true
+	E2E_CONFIG_PARAMS=$(E2E_CONFIG_PARAMS) E2E_PROFILE=$(E2E_PROFILE) go test -C test/e2e ./... --tags=e2e -v -vet=off -timeout 1h -count=1 -args -destroyStacks=true

test/common/common_e2e.go

@@ -23,12 +23,13 @@ import (
 	"k8s.io/client-go/tools/remotecommand"
 )
 
-var defaultLocalPulumiConfigs = runner.ConfigMap{
+var defaultPulumiConfigs = runner.ConfigMap{
 	"ddinfra:aws/defaultKeyPairName": auto.ConfigValue{Value: os.Getenv("AWS_KEYPAIR_NAME")},
 }
 var defaultCIPulumiConfigs = runner.ConfigMap{
-	"aws:skipCredentialsValidation": auto.ConfigValue{Value: "true"},
-	"aws:skipMetadataApiCheck":      auto.ConfigValue{Value: "false"},
+	"aws:skipCredentialsValidation":     auto.ConfigValue{Value: "true"},
+	"aws:skipMetadataApiCheck":          auto.ConfigValue{Value: "false"},
+	"ddinfra:aws/defaultPrivateKeyPath": auto.ConfigValue{Value: os.Getenv("AWS_PRIVATE_KEY_FILE")},
 }
 
 type E2EEnv struct {
@@ -101,6 +102,7 @@ func SetupConfig() (runner.ConfigMap, error) {
 	res := runner.ConfigMap{}
 	configs := parseE2EConfigParams()
 	if os.Getenv("E2E_PROFILE") == "ci" {
+		res.Merge(defaultPulumiConfigs)
 		res.Merge(defaultCIPulumiConfigs)
 	} else {
 		// use "local" E2E profile for local testing
@@ -111,7 +113,7 @@ func SetupConfig() (runner.ConfigMap, error) {
 		if !e2eApiKeyBool || !e2eAppKeyBool || !e2eAwsKeypairNameBool {
 			return nil, fmt.Errorf("missing required environment variables. Must set `E2E_API_KEY`, `E2E_APP_KEY`, and `AWS_KEYPAIR_NAME` for the local E2E profile")
 		} else {
-			res.Merge(defaultLocalPulumiConfigs)
+			res.Merge(defaultPulumiConfigs)
 		}
 	}
 

test/datadog-operator/baseline/Operator_Deployment_default.yaml

@@ -7,7 +7,7 @@ metadata:
   namespace: datadog-agent
   labels:
     app.kubernetes.io/name: datadog-operator
-    helm.sh/chart: datadog-operator-1.4.1
+    helm.sh/chart: datadog-operator-1.4.2
     app.kubernetes.io/instance: datadog-operator
     app.kubernetes.io/version: "1.3.0"
     app.kubernetes.io/managed-by: Helm

test/datadog-operator/baseline/Operator_Deployment_with_certManager.yaml

@@ -7,7 +7,7 @@ metadata:
   namespace: datadog-agent
   labels:
     app.kubernetes.io/name: datadog-operator
-    helm.sh/chart: datadog-operator-1.4.1
+    helm.sh/chart: datadog-operator-1.4.2
     app.kubernetes.io/instance: datadog-operator
     app.kubernetes.io/version: "1.3.0"
     app.kubernetes.io/managed-by: Helm

test/datadog/e2e_eks_test.go → test/e2e/datadog/e2e_eks_test.go

@@ -1,17 +1,21 @@
+//go:build e2e
+// +build e2e
+
 package datadog
 
 import (
 	"fmt"
 	"testing"
 
 	"github.com/DataDog/datadog-agent/test/new-e2e/runner"
-	"github.com/DataDog/helm-charts/test/common"
 	"github.com/pulumi/pulumi/sdk/v3/go/auto"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
+
+	"github.com/DataDog/helm-charts/test/common"
 )
 
 const namespace = "datadog"
@@ -30,8 +34,6 @@ func Test_E2E_AgentOnEKS(t *testing.T) {
 		"ddtestworkload:deploy":                      auto.ConfigValue{Value: "false"},
 		"ddinfra:aws/eks/linuxBottlerocketNodeGroup": auto.ConfigValue{Value: "false"},
 		"ddinfra:aws/eks/windowsNodeGroup":           auto.ConfigValue{Value: "false"},
-		// TODO: remove when upstream eks-pulumi bug is fixed https://github.com/pulumi/pulumi-eks/pull/886
-		"pulumi:disable-default-providers": auto.ConfigValue{Value: "[]"},
 	}
 	stackConfig.Merge(config)
 
@@ -48,14 +50,14 @@ func Test_E2E_AgentOnEKS(t *testing.T) {
 			kc := kubeconfig.Value.(map[string]interface{})
 			_, restConfig, k8sClient, err = common.NewClientFromKubeconfig(kc)
 			if err == nil {
-				verifyPods(t)
+				t.Run("Agent pods should be created", verifyPods)
 			}
 		} else {
 			err = fmt.Errorf("could not create Kubernetes client, cluster kubeconfig is nil")
 		}
 	}
 	if err != nil {
-		t.Skipf("Skipping test. Encountered problem creating or updating E2E stack: %s", err)
+		t.Errorf("Skipping tests. Encountered problem creating or updating E2E stack: %s", err)
 	}
 }
 
@@ -79,10 +81,15 @@ func verifyPods(t *testing.T) {
 		RestConfig: restConfig,
 	}
 
-	assertPodStatus(t, podExec, ddaPodList, "agent")
-	assertPodStatus(t, podExec, dcaPodList, "cluster-agent")
-	assertPodStatus(t, podExec, ccPodList, "agent")
-
+	t.Run("exec `agent status` for `agent` pod should not error", func(t *testing.T) {
+		assertPodStatus(t, podExec, ddaPodList, "agent")
+	})
+	t.Run("`exec `agent status` for `cluster-agent` pod should not error", func(t *testing.T) {
+		assertPodStatus(t, podExec, dcaPodList, "cluster-agent")
+	})
+	t.Run("exec `agent status` for `cluster-check-runner` pod should not error", func(t *testing.T) {
+		assertPodStatus(t, podExec, ccPodList, "agent")
+	})
 }
 
 func assertPodStatus(t *testing.T, podExec common.K8sExec, podList *v1.PodList, containerName string) {