e2e: collect kind logs on failure and destroy cluster (#359)

**Commit Message**

e2e: collect kind logs on failure and destroy cluster

When e2e tests succeed, destroy the kind cluster. When e2e tests fail,
keep the kind cluster up & running for troubleshooting, and collect the
logs to the `tests/e2e/logs` directory to facilitate log analysis.
These logs are also uploaded as CI artifacts when e2e fail, to help
troubleshoot why e2e tests could have failed on CI.

**Related Issues/PRs (if applicable)**

N/A

**Special notes for reviewers (if applicable)**

N/A

Signed-off-by: Ignasi Barrera <ignasi@tetrate.io>

.gitignore

@@ -10,6 +10,8 @@ out/
 # This is the placeholder for the access log file during extproc tests.
 ACCESS_LOG_PATH
 
+tests/e2e/logs
+
 # Files and directories to ignore in the site directory
 # dependencies
 site/node_modules

tests/e2e/e2e_test.go

@@ -25,6 +25,9 @@ const (
 	egLatest      = "v0.0.0-latest" // This defaults to the latest dev version.
 	egNamespace   = "envoy-gateway-system"
 	egDefaultPort = 10080
+
+	kindClusterName = "envoy-ai-gateway"
+	kindLogDir      = "./logs"
 )
 
 var egVersion = func() string {
@@ -35,17 +38,35 @@ var egVersion = func() string {
 	}
 }()
 
+// By default, kind logs are collected when the e2e tests fail. The TEST_KEEP_CLUSTER environment variable
+// can be set to "true" to preserve the logs and the kind cluster even if the tests pass.
+var keepCluster = func() bool {
+	v, _ := os.LookupEnv("TEST_KEEP_CLUSTER")
+	return v == "true"
+}()
+
 func initLog(msg string) {
 	fmt.Printf("\u001b[32m=== INIT LOG: %s\u001B[0m\n", msg)
 }
 
+func cleanupLog(msg string) {
+	fmt.Printf("\u001b[32m=== CLEANUP LOG: %s\u001B[0m\n", msg)
+}
+
 func TestMain(m *testing.M) {
 	ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(5*time.Minute))
 
 	// The following code sets up the kind cluster, installs the Envoy Gateway, and installs the AI Gateway.
 	// They must be idempotent and can be run multiple times so that we can run the tests multiple times on
 	// failures.
 
+	defer func() {
+		// If the setup or some tests panic, try to collect the cluster logs
+		if r := recover(); r != nil {
+			cleanupKindCluster(true)
+		}
+	}()
+
 	if err := initKindCluster(ctx); err != nil {
 		cancel()
 		panic(err)
@@ -68,28 +89,29 @@ func TestMain(m *testing.M) {
 
 	code := m.Run()
 	cancel()
+
+	cleanupKindCluster(code != 0)
+
 	os.Exit(code)
 }
 
 func initKindCluster(ctx context.Context) (err error) {
-	const kindClusterName = "envoy-ai-gateway"
-
 	initLog("Setting up the kind cluster")
 	start := time.Now()
 	defer func() {
 		elapsed := time.Since(start)
 		initLog(fmt.Sprintf("\tdone (took %.2fs in total)", elapsed.Seconds()))
 	}()
 
-	initLog("\tCreating kind cluster named envoy-ai-gateway")
+	initLog(fmt.Sprintf("\tCreating kind cluster named %s", kindClusterName))
 	cmd := exec.CommandContext(ctx, "go", "tool", "kind", "create", "cluster", "--name", kindClusterName)
 	out, err := cmd.CombinedOutput()
 	if err != nil && !bytes.Contains(out, []byte("already exist")) {
 		fmt.Printf("Error creating kind cluster: %s\n", out)
 		return
 	}
 
-	initLog("\tSwitching kubectl context to envoy-ai-gateway")
+	initLog(fmt.Sprintf("\tSwitching kubectl context to %s", kindClusterName))
 	cmd = exec.CommandContext(ctx, "go", "tool", "kind", "export", "kubeconfig", "--name", kindClusterName)
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
@@ -113,6 +135,23 @@ func initKindCluster(ctx context.Context) (err error) {
 	return nil
 }
 
+func cleanupKindCluster(testsFailed bool) {
+	if testsFailed || keepCluster {
+		cleanupLog("Collecting logs from the kind cluster")
+		cmd := exec.Command("go", "tool", "kind", "export", "logs", "--name", kindClusterName, kindLogDir)
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stderr
+		_ = cmd.Run()
+	}
+	if !testsFailed && !keepCluster {
+		cleanupLog("Destroying the kind cluster")
+		cmd := exec.Command("go", "tool", "kind", "delete", "cluster", "--name", kindClusterName)
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stderr
+		_ = cmd.Run()
+	}
+}
+
 // initEnvoyGateway initializes the Envoy Gateway in the kind cluster following the quickstart guide:
 // https://gateway.envoyproxy.io/latest/tasks/quickstart/
 func initEnvoyGateway(ctx context.Context) (err error) {