test(e2e): add pod logs capture for logging vector

zncdatadev · Dec 30, 2024 · 4c145e6 · 4c145e6
1 parent ed1e0af
commit 4c145e6
Show file tree

Hide file tree

Showing 5 changed files with 66 additions and 25 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -33,7 +33,7 @@ jobs:
     strategy:
       matrix:
         k8s-version: ['1.26.15', '1.27.16']
-        product_version: ['3.5.1', '3.5.2']
+        product_version: ['3.5.1'] # TODO: 3.5.2 image is constructing, should added after it is ready
       max-parallel: 2
     steps:
       - name: Clone the code

diff --git a/internal/controller/historyserver/deployment.go b/internal/controller/historyserver/deployment.go
@@ -20,6 +20,7 @@ import (
 	"github.com/zncdatadev/operator-go/pkg/util"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
+	"k8s.io/apimachinery/pkg/util/intstr"
 	ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
 
 	shsv1alpha1 "github.com/zncdatadev/spark-k8s-operator/api/v1alpha1"
@@ -135,6 +136,21 @@ func (b *DeploymentBuilder) getMainContainer(s3LogConfig *S3Logconfig) *builder.
 	containerBuilder.AddPorts(b.Ports)
 	containerBuilder.AddEnvVars(b.getMainContainerEnvVars())
 	containerBuilder.SetSecurityContext(0, 0, false)
+
+	probe := &corev1.Probe{
+		ProbeHandler: corev1.ProbeHandler{
+			TCPSocket: &corev1.TCPSocketAction{
+				Port: intstr.FromInt(18080),
+			},
+		},
+		InitialDelaySeconds: 10,
+		TimeoutSeconds:      5,
+		PeriodSeconds:       10,
+		SuccessThreshold:    1,
+	}
+	containerBuilder.SetReadinessProbe(probe)
+	containerBuilder.SetLivenessProbe(probe)
+
 	return containerBuilder
 }
 

diff --git a/test/e2e/logging/chainsaw-test.yaml b/test/e2e/logging/chainsaw-test.yaml
@@ -76,3 +76,16 @@ spec:
     cleanup:
     - sleep:
         duration: 50s
+    catch:
+      - script:
+          env:
+            - name: NAMESPACE
+              value: ($namespace)
+          content: |
+            kubectl -n $NAMESPACE describe pods
+      - podLogs:
+          selector: app.kubernetes.io/instance=vector-aggregator
+          tail: -1
+      - podLogs:
+          selector: app.kubernetes.io/instance=sparkhistory
+          tail: -1
diff --git a/test/e2e/smoke/override-pdb/chainsaw-test.yaml b/test/e2e/smoke/override-pdb/chainsaw-test.yaml
@@ -1,7 +1,7 @@
 apiVersion: chainsaw.kyverno.io/v1alpha1
 kind: Test
 metadata:
-  name: smoke-cluster-operation
+  name: smoke-override-pdb
 spec:
   bindings:
   - name: minio_user
@@ -15,21 +15,21 @@ spec:
   - name: cluster_paused
     value: false
   steps:
-  # - name: install minio
-  #   try:
-  #   - script:
-  #       env:
-  #         - name: NAMESPACE
-  #           value: ($namespace)
-  #         - name: MINIO_USER
-  #           value: ($minio_user)
-  #         - name: MINIO_PASSWORD
-  #           value: ($minio_password)
-  #       content: |
-  #         envsubst < ../../setup/minio-secret.yaml | kubectl apply -f -
-  #   - script:
-  #       content: |
-  #         bash ../../setup/minio-install.sh
+  - name: install minio
+    try:
+    - script:
+        env:
+          - name: NAMESPACE
+            value: ($namespace)
+          - name: MINIO_USER
+            value: ($minio_user)
+          - name: MINIO_PASSWORD
+            value: ($minio_password)
+        content: |
+          envsubst < ../../setup/minio-secret.yaml | kubectl apply -f -
+    - script:
+        content: |
+          bash ../../setup/minio-install.sh
   - name: install sparkhistoryserver
     try:
     # install sparkhistoryserver sparkhistoryserver, clusterOperation.stopped: false, clusterOperation.reconciliationPaused: false
@@ -78,8 +78,12 @@ spec:
                       cpu: 300m
                       memory: 526Mi
                     limits:
-                      cpu: 500m
-                      memory: 2Gi
+                      cpu: 350m
+                      memory: 1300Mi
+          status:
+            replicas: 2
+            readyReplicas: 2
+            availableReplicas: 2
   - name: test env overrides
     try:
     - script:
@@ -91,16 +95,17 @@ spec:
           kubectl -n $NAMESPACE get deployment test-sparkhistoryserver-node-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "node") | .env[] | select (.name == "COMMON_VAR" and .value == "group-value")'
           kubectl -n $NAMESPACE get deployment test-sparkhistoryserver-node-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "node") | .env[] | select (.name == "GROUP_VAR" and .value == "group-value")'
           kubectl -n $NAMESPACE get deployment test-sparkhistoryserver-node-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "node") | .env[] | select (.name == "ROLE_VAR" and .value == "role-value")'
-
   - name: run spark example
     try:
+    - sleep:
+        duration: 20s
     - script:
         bindings:
         - name: NAMESPACE
           value: ($namespace)
         content: |
           #!/bin/bash
-          POD=$(kubectl -n $NAMESPACE get pods -l app.kubernetes.io/name=sparkhistoryserver -o jsonpath='{.items[0].metadata.name}')
+          POD=$(kubectl -n $NAMESPACE get pods -l app.kubernetes.io/name=sparkhistoryserver --field-selector status.phase=Running -o jsonpath='{.items[0].metadata.name}')
           kubectl -n $NAMESPACE exec $POD -- /bin/bash -c '
           export AWS_ACCESS_KEY_ID=$(cat /kubedoop/secret/s3-credentials/ACCESS_KEY)
           export AWS_SECRET_ACCESS_KEY=$(cat /kubedoop/secret/s3-credentials/SECRET_KEY)
@@ -112,17 +117,19 @@ spec:
             --conf spark.hadoop.fs.s3a.endpoint=http://minio.chainsaw.svc.cluster.local:9000 \
             --conf spark.hadoop.fs.s3a.path.style.access=true \
             --conf spark.hadoop.fs.s3a.connection.ssl.enabled=false \
-            examples/src/main/python/pi.py 10
+            examples/src/main/python/pi.py 2
           '
   - name: verify history server
     try:
+    - sleep:
+        duration: 20s
     - script:
         bindings:
         - name: NAMESPACE
           value: ($namespace)
         content: |
           #!/bin/bash
-          POD=$(kubectl -n $NAMESPACE get pods -l app.kubernetes.io/name=sparkhistoryserver -o jsonpath='{.items[0].metadata.name}')
+          POD=$(kubectl -n $NAMESPACE get pods -l app.kubernetes.io/name=sparkhistoryserver --field-selector status.phase=Running -o jsonpath='{.items[0].metadata.name}')
           # Wait for the application to appear in history server
           for i in {1..30}; do
             APPS=$(kubectl -n $NAMESPACE exec $POD -- curl -s http://localhost:18080/api/v1/applications)
@@ -137,3 +144,8 @@ spec:
           done
           echo "Timeout waiting for application in history server"
           exit 1
+    catch:
+      - script:
+          content: |
+            #!/bin/bash
+            kubectl -n $NAMESPACE describe pods
diff --git a/test/e2e/smoke/override-pdb/sparkhistoryserver.yaml b/test/e2e/smoke/override-pdb/sparkhistoryserver.yaml
@@ -49,5 +49,5 @@ spec:
                     cpu: 300m
                     memory: 526Mi
                   limits:
-                    cpu: 500m
-                    memory: 2Gi
+                    cpu: 350m
+                    memory: 1300Mi