Enable datadog chart EKS E2E tests in CI (#1320)

* test e2e in ci * update e2e tests/bump infra ci image * run on push for testing * update integ tests, remove awscli update, add gh token * debug gh token * debug gh token * debug gh token * run e2e * add ssh key for ci * update tests * cleanup gitlab config * fix baselines * bump packages * test ci * bump runner resources * run on main/merge * add manual run
DataDog · Feb 21, 2024 · 96399e1 · 96399e1
1 parent 2cc1090
commit 96399e1
Show file tree

Hide file tree

Showing 7 changed files with 338 additions and 236 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -1,7 +1,3 @@
-variables:
-  # Do not modify this - must be the repository name for Kubernetes gitlab runners to run
-  KUBERNETES_SERVICE_ACCOUNT_OVERWRITE: helm-charts
-
 stages:
   - e2e
 
@@ -14,22 +10,32 @@ e2e:
           - charts/**/*.yaml
           - test/**/*
         compare_to: "refs/heads/main"
-  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/test-infra-definitions/runner:986d192b
+      when: always
+    - if: $CI_COMMIT_BRANCH != "main"
+      when: manual
+  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/test-infra-definitions/runner:95dca87f269a
   tags: ["arch:amd64"]
+  variables:
+    AWS_KEYPAIR_NAME: datadog-agent-ci
+    AWS_PRIVATE_KEY_FILE: $CI_PROJECT_DIR/ssh_key
+    KUBERNETES_CPU_REQUEST: 2
+    KUBERNETES_MEMORY_REQUEST: 4Gi
+    KUBERNETES_MEMORY_LIMIT: 12Gi
   before_script:
-    # Update awscli v2
-    - aws --version
-    - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
-    - unzip awscliv2.zip
-    - ./aws/install --update
-    - aws --version
-    # Setup AWS Credentials
     - mkdir -p ~/.aws
     - set +x
+    # Set GITHUB_TOKEN to avoid getting rate-limited when pulumi sdk downloads the kubernetes provider
+    - export GITHUB_TOKEN=$(aws ssm get-parameter --region us-east-1 --name ci.helm-charts.github_token --with-decryption --query "Parameter.Value" --out text)
+    # Configure AWS EC2 ssh key needed for create pulumi EKS environment
+    - aws ssm get-parameter --region us-east-1 --name ci.helm-charts.ssh_key --with-decryption --query "Parameter.Value" --out text > $AWS_PRIVATE_KEY_FILE
+    - set -x
+    # Without the newline ssh silently fails and moves on to try other auth methods
+    - echo "" >> $AWS_PRIVATE_KEY_FILE
+    - chmod 600 $AWS_PRIVATE_KEY_FILE
+    # Configure AWS profile
     - aws ssm get-parameter --region us-east-1 --name ci.helm-charts.e2e-agent-qa-profile --with-decryption --query "Parameter.Value" --out text >> ~/.aws/config
     - set -x
     - export AWS_PROFILE=agent-qa-ci
     - pulumi login "s3://dd-pulumi-state?region=us-east-1&awssdk=v2&profile=$AWS_PROFILE"
   script:
-    # TODO: enable running E2E tests when upstream pulumi-eks fix is merged https://github.com/pulumi/pulumi-eks/pull/886
-    - echo "Success!"
+    - make test-e2e
diff --git a/Makefile b/Makefile
@@ -46,15 +46,15 @@ vet:
 
 .PHONY: unit-test
 unit-test:
-	go test -C test ./... -count=1 -skip=E2E
+	go test -C test ./... -count=1
 
 .PHONY: update-test-baselines
 update-test-baselines:
-	go test -C test ./... -count=1 -skip=E2E -args -updateBaselines=true
+	go test -C test ./... -count=1 -args -updateBaselines=true
 
 .PHONY: integration-test
 integration-test:
-	go test -C test/integ --tags=integration -skip=E2E -count=1 -v
+	go test -C test/integ --tags=integration -count=1 -v
 
 # Running E2E tests locally:
 ## Must be connected to appgate
@@ -67,14 +67,14 @@ test-e2e: fmt vet e2e-test
 # aws-vault exec sso-agent-sandbox-account-admin -- make e2e-test
 .PHONY: e2e-test
 e2e-test:
-	E2E_CONFIG_PARAMS=$(E2E_CONFIG_PARAMS) E2E_PROFILE=$(E2E_PROFILE) go test -C test ./... -v -run=E2E -vet=off -timeout 1h -count=1
+	E2E_CONFIG_PARAMS=$(E2E_CONFIG_PARAMS) E2E_PROFILE=$(E2E_PROFILE) go test -C test/e2e ./... --tags=e2e -v -vet=off -timeout 1h -count=1
 
 # aws-vault exec sso-agent-sandbox-account-admin -- make e2e-test-preserve-stacks
 .PHONY: e2e-test-preserve-stacks
 e2e-test-preserve-stacks:
-	E2E_CONFIG_PARAMS=$(E2E_CONFIG_PARAMS) E2E_PROFILE=$(E2E_PROFILE) go test -C test ./... -v -run=E2E -vet=off -timeout 1h -count=1 -args -preserveStacks=true
+	E2E_CONFIG_PARAMS=$(E2E_CONFIG_PARAMS) E2E_PROFILE=$(E2E_PROFILE) go test -C test/e2e ./... --tags=e2e -v -vet=off -timeout 1h -count=1 -args -preserveStacks=true
 
 # aws-vault exec sso-agent-sandbox-account-admin -- make e2e-test-cleanup-stacks
 .PHONY: e2e-test-cleanup-stacks
 e2e-test-cleanup-stacks:
-	E2E_CONFIG_PARAMS=$(E2E_CONFIG_PARAMS) E2E_PROFILE=$(E2E_PROFILE) go test -C test ./... -v -run=E2E -vet=off -timeout 1h -count=1 -args -destroyStacks=true
+	E2E_CONFIG_PARAMS=$(E2E_CONFIG_PARAMS) E2E_PROFILE=$(E2E_PROFILE) go test -C test/e2e ./... --tags=e2e -v -vet=off -timeout 1h -count=1 -args -destroyStacks=true
diff --git a/test/common/common_e2e.go b/test/common/common_e2e.go
@@ -9,10 +9,10 @@ import (
 	"os"
 	"strings"
 
-	"github.com/DataDog/datadog-agent/test/new-e2e/utils/infra"
+	"github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/infra"
 	"github.com/DataDog/test-infra-definitions/scenarios/aws/eks"
 
-	"github.com/DataDog/datadog-agent/test/new-e2e/runner"
+	"github.com/DataDog/datadog-agent/test/new-e2e/pkg/runner"
 	"github.com/pulumi/pulumi/sdk/v3/go/auto"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -23,12 +23,13 @@ import (
 	"k8s.io/client-go/tools/remotecommand"
 )
 
-var defaultLocalPulumiConfigs = runner.ConfigMap{
+var defaultPulumiConfigs = runner.ConfigMap{
 	"ddinfra:aws/defaultKeyPairName": auto.ConfigValue{Value: os.Getenv("AWS_KEYPAIR_NAME")},
 }
 var defaultCIPulumiConfigs = runner.ConfigMap{
-	"aws:skipCredentialsValidation": auto.ConfigValue{Value: "true"},
-	"aws:skipMetadataApiCheck":      auto.ConfigValue{Value: "false"},
+	"aws:skipCredentialsValidation":     auto.ConfigValue{Value: "true"},
+	"aws:skipMetadataApiCheck":          auto.ConfigValue{Value: "false"},
+	"ddinfra:aws/defaultPrivateKeyPath": auto.ConfigValue{Value: os.Getenv("AWS_PRIVATE_KEY_FILE")},
 }
 
 type E2EEnv struct {
@@ -101,6 +102,7 @@ func SetupConfig() (runner.ConfigMap, error) {
 	res := runner.ConfigMap{}
 	configs := parseE2EConfigParams()
 	if os.Getenv("E2E_PROFILE") == "ci" {
+		res.Merge(defaultPulumiConfigs)
 		res.Merge(defaultCIPulumiConfigs)
 	} else {
 		// use "local" E2E profile for local testing
@@ -111,7 +113,7 @@ func SetupConfig() (runner.ConfigMap, error) {
 		if !e2eApiKeyBool || !e2eAppKeyBool || !e2eAwsKeypairNameBool {
 			return nil, fmt.Errorf("missing required environment variables. Must set `E2E_API_KEY`, `E2E_APP_KEY`, and `AWS_KEYPAIR_NAME` for the local E2E profile")
 		} else {
-			res.Merge(defaultLocalPulumiConfigs)
+			res.Merge(defaultPulumiConfigs)
 		}
 	}
 

diff --git a/test/datadog/e2e_eks_test.go → test/e2e/datadog/e2e_eks_test.go b/test/datadog/e2e_eks_test.go → test/e2e/datadog/e2e_eks_test.go
@@ -1,17 +1,21 @@
+//go:build e2e
+// +build e2e
+
 package datadog
 
 import (
 	"fmt"
 	"testing"
 
-	"github.com/DataDog/datadog-agent/test/new-e2e/runner"
-	"github.com/DataDog/helm-charts/test/common"
+	"github.com/DataDog/datadog-agent/test/new-e2e/pkg/runner"
 	"github.com/pulumi/pulumi/sdk/v3/go/auto"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
+
+	"github.com/DataDog/helm-charts/test/common"
 )
 
 const namespace = "datadog"
@@ -30,8 +34,6 @@ func Test_E2E_AgentOnEKS(t *testing.T) {
 		"ddtestworkload:deploy":                      auto.ConfigValue{Value: "false"},
 		"ddinfra:aws/eks/linuxBottlerocketNodeGroup": auto.ConfigValue{Value: "false"},
 		"ddinfra:aws/eks/windowsNodeGroup":           auto.ConfigValue{Value: "false"},
-		// TODO: remove when upstream eks-pulumi bug is fixed https://github.com/pulumi/pulumi-eks/pull/886
-		"pulumi:disable-default-providers": auto.ConfigValue{Value: "[]"},
 	}
 	stackConfig.Merge(config)
 
@@ -48,14 +50,14 @@ func Test_E2E_AgentOnEKS(t *testing.T) {
 			kc := kubeconfig.Value.(map[string]interface{})
 			_, restConfig, k8sClient, err = common.NewClientFromKubeconfig(kc)
 			if err == nil {
-				verifyPods(t)
+				t.Run("Agent pods should be created", verifyPods)
 			}
 		} else {
 			err = fmt.Errorf("could not create Kubernetes client, cluster kubeconfig is nil")
 		}
 	}
 	if err != nil {
-		t.Skipf("Skipping test. Encountered problem creating or updating E2E stack: %s", err)
+		t.Errorf("Skipping tests. Encountered problem creating or updating E2E stack: %s", err)
 	}
 }
 
@@ -79,10 +81,15 @@ func verifyPods(t *testing.T) {
 		RestConfig: restConfig,
 	}
 
-	assertPodStatus(t, podExec, ddaPodList, "agent")
-	assertPodStatus(t, podExec, dcaPodList, "cluster-agent")
-	assertPodStatus(t, podExec, ccPodList, "agent")
-
+	t.Run("exec `agent status` for `agent` pod should not error", func(t *testing.T) {
+		assertPodStatus(t, podExec, ddaPodList, "agent")
+	})
+	t.Run("`exec `agent status` for `cluster-agent` pod should not error", func(t *testing.T) {
+		assertPodStatus(t, podExec, dcaPodList, "cluster-agent")
+	})
+	t.Run("exec `agent status` for `cluster-check-runner` pod should not error", func(t *testing.T) {
+		assertPodStatus(t, podExec, ccPodList, "agent")
+	})
 }
 
 func assertPodStatus(t *testing.T, podExec common.K8sExec, podList *v1.PodList, containerName string) {

diff --git a/test/datadog/testdatadog_test.go → test/e2e/datadog/testdatadog_test.go b/test/datadog/testdatadog_test.go → test/e2e/datadog/testdatadog_test.go