ci: Add log template to PR and Load Test Pipeline (Azure#2264)

* Initial commit * add: log paths and label to priv. daemonset * add: log template * add: log template to load-test yamls * remove: kubeconfig calls * add: capture failed pods on job failure * add: Linux state files * add: Windows state files * style: change terminal output * add: log template to PR pipeline * fix: rebase * style: add comments to log-template * chore: Addressing Comments * add: sub-directories * ci: Only call log-template on fail for PR
nddq · Oct 3, 2023 · 64f01b2 · 64f01b2
1 parent ad3329a
commit 64f01b2
Show file tree

Hide file tree

Showing 16 changed files with 474 additions and 7 deletions.
diff --git a/.pipelines/cni/cilium/cilium-overlay-load-test-template.yaml b/.pipelines/cni/cilium/cilium-overlay-load-test-template.yaml
@@ -84,6 +84,7 @@ stages:
               os: linux
               iterations: ${ITERATIONS_CILIUM}
               nodeCount: ${{ parameters.nodeCount }}
+              cni: cilium
           - template: ../load-test-templates/validate-state-template.yaml
             parameters:
               clusterName: ${{ parameters.clusterName }}-$(commitID)
@@ -94,8 +95,10 @@ stages:
           - template: ../load-test-templates/restart-node-template.yaml
             parameters:
               clusterName: ${{ parameters.clusterName }}-$(commitID)
+              os: ${{ parameters.os }}
               nodeCount: ${{ parameters.nodeCount }}
               scaleup: ${SCALEUP_CILIUM}
+              cni: cilium
           - template: ../load-test-templates/validate-state-template.yaml
             parameters:
               clusterName: ${{ parameters.clusterName }}-$(commitID)
@@ -108,8 +111,10 @@ stages:
           - template: ../load-test-templates/restart-cns-template.yaml
             parameters:
               clusterName: ${{ parameters.clusterName }}-$(commitID)
+              os: ${{ parameters.os }}
               scaleup: ${SCALEUP_CILIUM}
               nodeCount: ${{ parameters.nodeCount }}
+              cni: cilium
       - job: cni_tests
         displayName: "Cilium Test"
         dependsOn: restart_cns
@@ -141,4 +146,18 @@ stages:
             retryCountOnTaskFailure: 6
             name: "CiliumConnectivityTests"
             displayName: "Run Cilium Connectivity Tests"
-
+      - job: logs
+        displayName: "Log Failure"
+        dependsOn:
+          - deploy_cilium_components
+          - deploy_pods
+          - restart_nodes
+          - restart_cns
+          - cni_tests
+        condition: failed()
+        steps:
+          - template: ../../templates/log-template.yaml
+            parameters:
+              clusterName: ${{ parameters.clusterName }}-$(commitID)
+              os: linux
+              cni: cilium
diff --git a/.pipelines/cni/cilium/nightly-release-test.yml b/.pipelines/cni/cilium/nightly-release-test.yml
@@ -88,11 +88,23 @@ stages:
             name: "cilium_nightly"
             testDropgz: ""
             clusterName: ciliumnightly-$(commitID)
+      - job: logs
+        displayName: "Log Failure"
+        dependsOn:
+          - cilium_nightly
+        condition: failed()
+        steps:
+          - template: ../../templates/log-template.yaml
+            parameters:
+              clusterName: ciliumnightly-$(commitID)
+              os: linux
+              cni: cilium
       - job: delete
         displayName: Delete Cluster
         condition: always()
         dependsOn:
           - cilium_nightly
+          - logs
         pool:
           name: "$(BUILD_POOL_NAME_DEFAULT)"
         strategy:

diff --git a/.pipelines/cni/load-test-templates/pod-deployment-template.yaml b/.pipelines/cni/load-test-templates/pod-deployment-template.yaml
@@ -4,6 +4,7 @@ parameters:
   os: ""
   iterations: 4
   nodeCount: 10
+  cni: ""
 
 steps:
   - task: AzureCLI@1
@@ -20,3 +21,10 @@ steps:
         cd test/integration/load
         scale=$(( ${{ parameters.scaleup }} * ${{ parameters.nodeCount }} ))
         ITERATIONS=${{ parameters.iterations }} SCALE_UP=$scale OS_TYPE=${{ parameters.os }} go test -timeout 30m -tags load -run ^TestLoad$
+
+  - template: ../../templates/log-template.yaml
+    parameters:
+      clusterName: ${{ parameters.clusterName }}
+      logType: scaleTest
+      os: ${{ parameters.os }}
+      cni: ${{ parameters.cni }}
diff --git a/.pipelines/cni/load-test-templates/restart-cns-template.yaml b/.pipelines/cni/load-test-templates/restart-cns-template.yaml
@@ -3,6 +3,7 @@ parameters:
   cni: "cilium"
   scaleup: 100
   nodeCount: 10
+  os: ""
 
 steps:
   - task: AzureCLI@1
@@ -32,3 +33,10 @@ steps:
     name: "restartCNS"
     displayName: "Restart CNS and Validate pods"
     retryCountOnTaskFailure: 3
+
+  - template: ../../templates/log-template.yaml
+    parameters:
+      clusterName: ${{ parameters.clusterName }}
+      logType: restartCNS
+      os: ${{ parameters.os }}
+      cni: ${{ parameters.cni }}
diff --git a/.pipelines/cni/load-test-templates/restart-node-template.yaml b/.pipelines/cni/load-test-templates/restart-node-template.yaml
@@ -3,6 +3,7 @@ parameters:
   nodeCount: 10
   scaleup: 100
   os: "linux"
+  cni: ""
 
 steps:
   - task: AzureCLI@1
@@ -30,3 +31,10 @@ steps:
         REPLICAS=$scale OS_TYPE=${{ parameters.os }} go test -count 1 -timeout 30m -tags load -run ^TestScaleDeployment$
     name: "RestartNodes"
     displayName: "Restart Nodes"
+
+  - template: ../../templates/log-template.yaml
+    parameters:
+      clusterName: ${{ parameters.clusterName }}
+      logType: restartNode
+      os: ${{ parameters.os }}
+      cni: ${{ parameters.cni }}
diff --git a/.pipelines/cni/singletenancy/cniv1-template.yaml b/.pipelines/cni/singletenancy/cniv1-template.yaml
@@ -105,6 +105,7 @@ stages:
               os: ${{ parameters.os }}
               iterations: ${{ parameters.iterations }}
               nodeCount: ${{ parameters.nodeCount }}
+              cni: cniv1
           - template: ../load-test-templates/validate-state-template.yaml
             parameters:
               clusterName: ${{ parameters.clusterName }}-$(commitID)
@@ -120,6 +121,7 @@ stages:
               os: ${{ parameters.os }}
               nodeCount: ${{ parameters.nodeCount }}
               scaleup: ${{ parameters.scaleup }}
+              cni: cniv1
           - template: ../load-test-templates/validate-state-template.yaml
             parameters:
               clusterName: ${{ parameters.clusterName }}-$(commitID)
@@ -156,6 +158,21 @@ stages:
           hybridWin: true
           service: true
           hostport: true
+      - job: logs
+        displayName: "Log Failure"
+        dependsOn:
+          - update_cni
+          - npm_k8se2e
+          - deploy_pods
+          - restart_nodes
+          - recover
+          - cni_${{ parameters.os }}
+        condition: failed()
+        steps:
+          - template: ../../templates/log-template.yaml
+            parameters:
+              clusterName: ${{ parameters.clusterName }}-$(commitID)
+              os: ${{ parameters.os }}
 
   - ${{ if eq(parameters.os, 'windows') }}:
     - stage: ${{ parameters.name }}_HNS
@@ -240,3 +257,17 @@ stages:
             hybridWin: true
             service: true
             hostport: true
+        - job: logs
+          displayName: "Log Failure"
+          dependsOn:
+            - restart_hns
+            - restart_nodes
+            - deploy_pods
+            - recover
+            - cni_${{ parameters.os }}
+          condition: failed()
+          steps:
+            - template: ../../templates/log-template.yaml
+              parameters:
+                clusterName: ${{ parameters.clusterName }}-$(commitID)
+                os: ${{ parameters.os }}
diff --git a/.pipelines/cni/singletenancy/linux-cniv2-template.yaml b/.pipelines/cni/singletenancy/linux-cniv2-template.yaml
@@ -108,6 +108,7 @@ stages:
               os: ${{ parameters.os }}
               iterations: ${ITERATIONS_LINUX}
               nodeCount: ${{ parameters.nodeCount }}
+              cni: cniv2
           - template: ../load-test-templates/validate-state-template.yaml
             parameters:
               clusterName: ${{ parameters.clusterName }}-$(commitID)
@@ -120,8 +121,10 @@ stages:
           - template: ../load-test-templates/restart-node-template.yaml
             parameters:
               clusterName: ${{ parameters.clusterName }}-$(commitID)
+              os: ${{ parameters.os }}
               nodeCount: ${{ parameters.nodeCount }}
               scaleup: ${SCALEUP_LINUX}
+              cni: cniv2
           - template: ../load-test-templates/validate-state-template.yaml
             parameters:
               clusterName: ${{ parameters.clusterName }}-$(commitID)
@@ -135,6 +138,7 @@ stages:
           - template: ../load-test-templates/restart-cns-template.yaml
             parameters:
               clusterName: ${{ parameters.clusterName }}-$(commitID)
+              os: ${{ parameters.os }}
               cni: cniv2
               scaleup: ${SCALEUP_LINUX}
               nodeCount: ${{ parameters.nodeCount }}
@@ -167,4 +171,20 @@ stages:
           portforward: true
           service: true
           hostport: true
-
+      - job: logs
+        displayName: "Log Failure"
+        dependsOn:
+          - integration
+          - npm_k8se2e
+          - deploy_pods
+          - restart_nodes
+          - restart_cns
+          - recover
+          - cni_${{ parameters.os }}
+        condition: failed()
+        steps:
+          - template: ../../templates/log-template.yaml
+            parameters:
+              clusterName: ${{ parameters.clusterName }}-$(commitID)
+              os: ${{ parameters.os }}
+              cni: cniv2
diff --git a/.pipelines/npm/npm-cni-integration-test.yaml b/.pipelines/npm/npm-cni-integration-test.yaml
@@ -126,9 +126,8 @@ jobs:
         continueOnError: ${{ parameters.continueOnError }}
 
       - bash: |
-          npmLogs=$(System.DefaultWorkingDirectory)/Attempt_#$(System.StageAttempt)/npmLogs_${{ parameters.clusterName }}
+          npmLogs=$(System.DefaultWorkingDirectory)/${{ parameters.clusterName }}_npmLogs_Attempt_#$(System.StageAttempt)
           mkdir -p $npmLogs
-          cp ./kubeconfig $npmLogs/kubeconfig
 
           kubectl get pods -n kube-system | grep npm
           npmPodList=`kubectl get pods -n kube-system | grep npm | awk '{print $1}'`
@@ -139,8 +138,8 @@ jobs:
         displayName: Generate NPM pod logs
         condition: always()
 
-      - publish: $(System.DefaultWorkingDirectory)/Attempt_#$(System.StageAttempt)/npmLogs_${{ parameters.clusterName }}
+      - publish: $(System.DefaultWorkingDirectory)/${{ parameters.clusterName }}_npmLogs_Attempt_#$(System.StageAttempt)
         condition: always()
-        artifact: npmLogs_${{ parameters.clusterName }}
+        artifact: ${{ parameters.clusterName }}_npmLogs_Attempt_#$(System.StageAttempt)
         name: npmLogs
         displayName: Publish NPM pod logs
diff --git a/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml
@@ -60,3 +60,15 @@ stages:
               name: ${{ parameters.name }}
               testDropgz: ${{ parameters.testDropgz }}
               clusterName: ${{ parameters.clusterName }}-$(commitID)
+
+      - job: logs
+        displayName: "Log Failure"
+        dependsOn:
+          - ${{ parameters.name }}
+        condition: failed()
+        steps:
+          - template: ../../templates/log-template.yaml
+            parameters:
+              clusterName: ${{ parameters.clusterName }}-$(commitID)
+              os: ${{ parameters.os }}
+              cni: cniv2
diff --git a/.pipelines/singletenancy/aks/e2e-job-template.yaml b/.pipelines/singletenancy/aks/e2e-job-template.yaml
@@ -79,3 +79,16 @@ stages:
           service: true
           hostport: true
           dependsOn: ${{ parameters.name }}
+
+      - job: logs
+        displayName: "Log Failure"
+        dependsOn:
+          - ${{ parameters.name }}
+          - cni_${{ parameters.os }}
+        condition: failed()
+        steps:
+          - template: ../../templates/log-template.yaml
+            parameters:
+              clusterName: ${{ parameters.clusterName }}-$(commitID)
+              os: ${{ parameters.os }}
+              cni: cniv1
diff --git a/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-job-template.yaml b/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-job-template.yaml
@@ -57,3 +57,15 @@ stages:
               name: ${{ parameters.name }}
               testDropgz: ${{ parameters.testDropgz }}
               clusterName: ${{ parameters.clusterName }}-$(commitID)
+
+      - job: logs
+        displayName: "Log Failure"
+        dependsOn:
+          - ${{ parameters.name }}
+        condition: failed()
+        steps:
+          - template: ../../templates/log-template.yaml
+            parameters:
+              clusterName: ${{ parameters.clusterName }}-$(commitID)
+              os: ${{ parameters.os }}
+              cni: cniv2
diff --git a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-job-template.yaml b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-job-template.yaml
@@ -56,3 +56,15 @@ stages:
               name: ${{ parameters.name }}
               testDropgz: ${{ parameters.testDropgz }}
               clusterName: ${{ parameters.clusterName }}-$(commitID)
+
+      - job: logs
+        displayName: "Log Failure"
+        dependsOn:
+          - ${{ parameters.name }}
+        condition: failed()
+        steps:
+          - template: ../../templates/log-template.yaml
+            parameters:
+              clusterName: ${{ parameters.clusterName }}-$(commitID)
+              os: ${{ parameters.os }}
+              cni: cilium
diff --git a/.pipelines/singletenancy/cilium/cilium-e2e-job-template.yaml b/.pipelines/singletenancy/cilium/cilium-e2e-job-template.yaml
@@ -60,3 +60,15 @@ stages:
               name: ${{ parameters.name }}
               testDropgz: ${{ parameters.testDropgz }}
               clusterName: ${{ parameters.clusterName }}-$(commitID)
+
+      - job: logs
+        displayName: "Log Failure"
+        dependsOn:
+          - ${{ parameters.name }}
+        condition: failed()
+        steps:
+          - template: ../../templates/log-template.yaml
+            parameters:
+              clusterName: ${{ parameters.clusterName }}-$(commitID)
+              os: ${{ parameters.os }}
+              cni: cilium
diff --git a/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-job-template.yaml b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-job-template.yaml
@@ -80,3 +80,17 @@ stages:
           portforward: true
           service: true
           hybridWin: true
+
+      - job: logs
+        displayName: "Log Failure"
+        dependsOn:
+          - ${{ parameters.name }}
+          - cni_${{ parameters.os }}
+          - cni_windows
+        condition: failed()
+        steps:
+          - template: ../../templates/log-template.yaml
+            parameters:
+              clusterName: ${{ parameters.clusterName }}-$(commitID)
+              os: ${{ parameters.os }}
+              cni: cniv2