From 2b2d0ed5f0b736afeec9f1d607a5176c953628dd Mon Sep 17 00:00:00 2001
From: Ignasi Barrera <ignasi@tetrate.io>
Date: Wed, 19 Feb 2025 02:00:11 +0100
Subject: [PATCH] e2e: collect kind logs on failure and destroy cluster (#359)

**Commit Message**

e2e: collect kind logs on failure and destroy cluster

When e2e tests succeed, destroy the kind cluster. When e2e tests fail,
keep the kind cluster up & running for troubleshooting, and collect the
logs to the `tests/e2e/logs` directory to facilitate log analysis.
These logs are also uploaded as CI artifacts when e2e fail, to help
troubleshoot why e2e tests could have failed on CI.

**Related Issues/PRs (if applicable)**

N/A

**Special notes for reviewers (if applicable)**

N/A

Signed-off-by: Ignasi Barrera <ignasi@tetrate.io>
---
 .gitignore            |  2 ++
 tests/e2e/e2e_test.go | 47 +++++++++++++++++++++++++++++++++++++++----
 2 files changed, 45 insertions(+), 4 deletions(-)

diff --git a/.gitignore b/.gitignore
index ba97f8b9..c720a00d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -10,6 +10,8 @@ out/
 # This is the placeholder for the access log file during extproc tests.
 ACCESS_LOG_PATH
 
+tests/e2e/logs
+
 # Files and directories to ignore in the site directory
 # dependencies
 site/node_modules
diff --git a/tests/e2e/e2e_test.go b/tests/e2e/e2e_test.go
index 15989775..1329ca5e 100644
--- a/tests/e2e/e2e_test.go
+++ b/tests/e2e/e2e_test.go
@@ -25,6 +25,9 @@ const (
 	egLatest      = "v0.0.0-latest" // This defaults to the latest dev version.
 	egNamespace   = "envoy-gateway-system"
 	egDefaultPort = 10080
+
+	kindClusterName = "envoy-ai-gateway"
+	kindLogDir      = "./logs"
 )
 
 var egVersion = func() string {
@@ -35,10 +38,21 @@ var egVersion = func() string {
 	}
 }()
 
+// By default, kind logs are collected when the e2e tests fail. The TEST_KEEP_CLUSTER environment variable
+// can be set to "true" to preserve the logs and the kind cluster even if the tests pass.
+var keepCluster = func() bool {
+	v, _ := os.LookupEnv("TEST_KEEP_CLUSTER")
+	return v == "true"
+}()
+
 func initLog(msg string) {
 	fmt.Printf("\u001b[32m=== INIT LOG: %s\u001B[0m\n", msg)
 }
 
+func cleanupLog(msg string) {
+	fmt.Printf("\u001b[32m=== CLEANUP LOG: %s\u001B[0m\n", msg)
+}
+
 func TestMain(m *testing.M) {
 	ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(5*time.Minute))
 
@@ -46,6 +60,13 @@ func TestMain(m *testing.M) {
 	// They must be idempotent and can be run multiple times so that we can run the tests multiple times on
 	// failures.
 
+	defer func() {
+		// If the setup or some tests panic, try to collect the cluster logs
+		if r := recover(); r != nil {
+			cleanupKindCluster(true)
+		}
+	}()
+
 	if err := initKindCluster(ctx); err != nil {
 		cancel()
 		panic(err)
@@ -68,12 +89,13 @@ func TestMain(m *testing.M) {
 
 	code := m.Run()
 	cancel()
+
+	cleanupKindCluster(code != 0)
+
 	os.Exit(code)
 }
 
 func initKindCluster(ctx context.Context) (err error) {
-	const kindClusterName = "envoy-ai-gateway"
-
 	initLog("Setting up the kind cluster")
 	start := time.Now()
 	defer func() {
@@ -81,7 +103,7 @@ func initKindCluster(ctx context.Context) (err error) {
 		initLog(fmt.Sprintf("\tdone (took %.2fs in total)", elapsed.Seconds()))
 	}()
 
-	initLog("\tCreating kind cluster named envoy-ai-gateway")
+	initLog(fmt.Sprintf("\tCreating kind cluster named %s", kindClusterName))
 	cmd := exec.CommandContext(ctx, "go", "tool", "kind", "create", "cluster", "--name", kindClusterName)
 	out, err := cmd.CombinedOutput()
 	if err != nil && !bytes.Contains(out, []byte("already exist")) {
@@ -89,7 +111,7 @@ func initKindCluster(ctx context.Context) (err error) {
 		return
 	}
 
-	initLog("\tSwitching kubectl context to envoy-ai-gateway")
+	initLog(fmt.Sprintf("\tSwitching kubectl context to %s", kindClusterName))
 	cmd = exec.CommandContext(ctx, "go", "tool", "kind", "export", "kubeconfig", "--name", kindClusterName)
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
@@ -113,6 +135,23 @@ func initKindCluster(ctx context.Context) (err error) {
 	return nil
 }
 
+func cleanupKindCluster(testsFailed bool) {
+	if testsFailed || keepCluster {
+		cleanupLog("Collecting logs from the kind cluster")
+		cmd := exec.Command("go", "tool", "kind", "export", "logs", "--name", kindClusterName, kindLogDir)
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stderr
+		_ = cmd.Run()
+	}
+	if !testsFailed && !keepCluster {
+		cleanupLog("Destroying the kind cluster")
+		cmd := exec.Command("go", "tool", "kind", "delete", "cluster", "--name", kindClusterName)
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stderr
+		_ = cmd.Run()
+	}
+}
+
 // initEnvoyGateway initializes the Envoy Gateway in the kind cluster following the quickstart guide:
 // https://gateway.envoyproxy.io/latest/tasks/quickstart/
 func initEnvoyGateway(ctx context.Context) (err error) {