From 69a174d36cadde804fc36ae58251d6b0fb976f47 Mon Sep 17 00:00:00 2001 From: Denys Fedoryshchenko Date: Thu, 9 Nov 2023 11:18:40 +0200 Subject: [PATCH] aks/: Update Azure deployment files Update Azure deployment files, to automate deployment. Signed-off-by: Denys Fedoryshchenko --- kube/aks/README.md | 6 ++ kube/aks/ingress.yaml | 25 +++++ kube/aks/kernelci-secrets.toml.example | 49 +++++++++ kube/aks/kernelci.toml | 16 --- kube/aks/lava-callback.yaml | 40 ++++++++ kube/aks/monitor.yaml | 48 +++++---- kube/aks/nodehandlers.yaml | 92 +++++++++++++++++ kube/aks/restart.sh | 64 ------------ kube/aks/scheduler-k8s.yaml | 113 +++++++-------------- kube/aks/scheduler-lava.yaml | 90 +++++++---------- kube/aks/tarball.yaml | 134 ++++++++----------------- kube/aks/timeout.yaml | 70 ------------- kube/aks/trigger.yaml | 48 +++++---- 13 files changed, 376 insertions(+), 419 deletions(-) create mode 100644 kube/aks/README.md create mode 100644 kube/aks/ingress.yaml create mode 100644 kube/aks/kernelci-secrets.toml.example delete mode 100644 kube/aks/kernelci.toml create mode 100644 kube/aks/lava-callback.yaml create mode 100644 kube/aks/nodehandlers.yaml delete mode 100755 kube/aks/restart.sh delete mode 100644 kube/aks/timeout.yaml diff --git a/kube/aks/README.md b/kube/aks/README.md new file mode 100644 index 000000000..c08b133e3 --- /dev/null +++ b/kube/aks/README.md @@ -0,0 +1,6 @@ +# Pipeline Kubernetes manifest files + +## Usage + +This files designed to be used by api-pipeline-deploy.sh script from [kernelci-deploy](https://github.com/kernelci/kernelci-deploy) repository. +Additional documentation can be found in [kernelci-deploy README](https://github.com/kernelci/kernelci-deploy/kubernetes/README.md). diff --git a/kube/aks/ingress.yaml b/kube/aks/ingress.yaml new file mode 100644 index 000000000..03fa66b93 --- /dev/null +++ b/kube/aks/ingress.yaml @@ -0,0 +1,25 @@ +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: pipeline-ingress + namespace: kernelci-pipeline-testns + annotations: + cert-manager.io/cluster-issuer: all-issuer +spec: + ingressClassName: nginx-pipeline + tls: + - hosts: + - kernelci-pipeline-staging.eastus.cloudapp.azure.com + secretName: pipeline-tls + rules: + - host: kernelci-pipeline-staging.eastus.cloudapp.azure.com + http: + paths: + - backend: + service: + name: lava-callback + port: + number: 8000 + path: / + pathType: Prefix diff --git a/kube/aks/kernelci-secrets.toml.example b/kube/aks/kernelci-secrets.toml.example new file mode 100644 index 000000000..233148064 --- /dev/null +++ b/kube/aks/kernelci-secrets.toml.example @@ -0,0 +1,49 @@ +[DEFAULT] +api_config = "staging" +storage_config = "staging-azure" +verbose = true + +[trigger] +poll_period = 3600 +startup_delay = 3 +timeout = 60 + +[tarball] +kdir = "/home/kernelci/data/src/linux" +output = "/home/kernelci/data/output" + +[scheduler] +output = "/home/kernelci/data/output" +runtime_config = "k8s-gke-eu-west4" + +[monitor] + +[send_kcidb] +kcidb_topic_name = "playground_kcidb_new" +kcidb_project_id = "kernelci-production" +origin = "kernelci_api" + +[test_report] +smtp_host = "smtp.gmail.com" +smtp_port = 465 +email_sender = "bot@kernelci.org" +email_recipient = "kernelci-results-staging@groups.io" + +[timeout] + +[regression_tracker] + +[storage.staging] +storage_cred = "/home/kernelci/data/ssh/id_rsa_tarball" + +[storage.staging-azure] +storage_cred = "" + +[storage.early-access-azure] +storage_cred = "" + +[runtime.lava-collabora] +runtime_token = "" + +[runtime.lava-collabora-early-access] +runtime_token = "" diff --git a/kube/aks/kernelci.toml b/kube/aks/kernelci.toml deleted file mode 100644 index 07204365f..000000000 --- a/kube/aks/kernelci.toml +++ /dev/null @@ -1,16 +0,0 @@ -[DEFAULT] -api_config = "early-access" -verbose = true -storage_config = "early-access-azure" - -[trigger] -poll_period = 3600 -startup_delay = 3 -build_configs = "mainline" - -[tarball] -kdir = "/home/kernelci/pipeline/data/src/linux" -output = "/home/kernelci/pipeline/data/output" - -[scheduler] -output = "/home/kernelci/pipeline/data/output" diff --git a/kube/aks/lava-callback.yaml b/kube/aks/lava-callback.yaml new file mode 100644 index 000000000..dc8461bc9 --- /dev/null +++ b/kube/aks/lava-callback.yaml @@ -0,0 +1,40 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: lava-callback + namespace: kernelci-pipeline-testns +spec: + replicas: 1 + selector: + matchLabels: + app: lava-callback + template: + metadata: + labels: + app: lava-callback + spec: + containers: + - name: lava-callback + image: denysfcollabora/pipeline-lava-callback + imagePullPolicy: Always + command: + - python3 + - src/lava_callback.py + env: + - name: KCI_API_TOKEN + valueFrom: + secretKeyRef: + name: kernelci-api-token + key: token +--- +apiVersion: v1 +kind: Service +metadata: + name: lava-callback + namespace: kernelci-pipeline-testns +spec: + ports: + - port: 80 + targetPort: 8000 + selector: + app: lava-callback diff --git a/kube/aks/monitor.yaml b/kube/aks/monitor.yaml index 83d520fcf..60549ed05 100644 --- a/kube/aks/monitor.yaml +++ b/kube/aks/monitor.yaml @@ -1,25 +1,29 @@ -# SPDX-License-Identifier: LGPL-2.1-or-later -# -# Copyright (C) 2023 Collabora Limited -# Author: Guillaume Tucker - -apiVersion: v1 -kind: Pod +apiVersion: apps/v1 +kind: Deployment metadata: name: monitor - namespace: kernelci-pipeline + namespace: kernelci-pipeline-testns spec: - containers: - - name: monitor - image: kernelci/pipeline - imagePullPolicy: Always - command: - - ./src/monitor.py - - --settings=/home/kernelci/pipeline/kube/aks/kernelci.toml - - run - env: - - name: KCI_API_TOKEN - valueFrom: - secretKeyRef: - name: kernelci-api-token - key: token + replicas: 1 + selector: + matchLabels: + app: monitor + template: + metadata: + labels: + app: monitor + spec: + containers: + - name: monitor + image: kernelci/pipeline + imagePullPolicy: Always + command: + - ./src/monitor.py + - --settings=/home/kernelci/pipeline/kube/aks/kernelci.toml + - run + env: + - name: KCI_API_TOKEN + valueFrom: + secretKeyRef: + name: kernelci-api-token + key: token diff --git a/kube/aks/nodehandlers.yaml b/kube/aks/nodehandlers.yaml new file mode 100644 index 000000000..24bdd7a33 --- /dev/null +++ b/kube/aks/nodehandlers.yaml @@ -0,0 +1,92 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: timeout + namespace: kernelci-pipeline-testns +spec: + replicas: 1 + selector: + matchLabels: + app: timeout + template: + metadata: + labels: + app: timeout + spec: + containers: + - name: timeout + image: kernelci/pipeline + imagePullPolicy: Always + command: + - ./src/timeout.py + - --settings=/home/kernelci/pipeline/kube/aks/kernelci.toml + - run + - --mode=timeout + env: + - name: KCI_API_TOKEN + valueFrom: + secretKeyRef: + name: kernelci-api-token + key: token +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: closing + namespace: kernelci-pipeline-testns +spec: + replicas: 1 + selector: + matchLabels: + app: closing + template: + metadata: + labels: + app: closing + spec: + containers: + - name: timeout + image: kernelci/pipeline + imagePullPolicy: Always + command: + - ./src/timeout.py + - --settings=/home/kernelci/pipeline/kube/aks/kernelci.toml + - run + - --mode=closing + env: + - name: KCI_API_TOKEN + valueFrom: + secretKeyRef: + name: kernelci-api-token + key: token +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: holdoff + namespace: kernelci-pipeline-testns +spec: + replicas: 1 + selector: + matchLabels: + app: holdoff + template: + metadata: + labels: + app: holdoff + spec: + containers: + - name: timeout + image: kernelci/pipeline + imagePullPolicy: Always + command: + - ./src/timeout.py + - --settings=/home/kernelci/pipeline/kube/aks/kernelci.toml + - run + - --mode=holdoff + env: + - name: KCI_API_TOKEN + valueFrom: + secretKeyRef: + name: kernelci-api-token + key: token diff --git a/kube/aks/restart.sh b/kube/aks/restart.sh deleted file mode 100755 index 617bc1982..000000000 --- a/kube/aks/restart.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/bash -# -# SPDX-License-Identifier: LGPL-2.1-or-later -# -# Copyright (C) 2023 Collabora Limited -# Author: Guillaume Tucker - -set -e - -first="\ -tarball \ -timeout \ -monitor \ -scheduler-k8s \ -scheduler-lava \ -" - -second="\ -trigger \ -" - -stop_pods() { - local pods=$(\ - kubectl get pods -o name \ - | while read line; do - echo $line | cut -d\/ -f2 - done \ - ) - - for pod in $pods; do - echo "* Stopping: $pod" - kubectl delete pod $pod --wait=false - done - - for pod in $pods; do - echo "* Waiting to stop: $pod" - kubectl wait --for=delete pod $pod - done - - return 0 -} - -start() { - local items=$1 - - for item in $items; do - echo "* Applying $item" - kubectl apply -f "$item".yaml --wait=false - done - - for item in $items; do - echo "* Waiting to start: $item" - kubectl wait --for=condition=Ready --timeout=1200s pod $item - done - - return 0 -} - -stop_pods -start "$first" -start "$second" -kubectl get pods - -exit 0 diff --git a/kube/aks/scheduler-k8s.yaml b/kube/aks/scheduler-k8s.yaml index 54b0dd7da..f3593f9bc 100644 --- a/kube/aks/scheduler-k8s.yaml +++ b/kube/aks/scheduler-k8s.yaml @@ -1,82 +1,37 @@ -# SPDX-License-Identifier: LGPL-2.1-or-later -# -# Copyright (C) 2023 Collabora Limited -# Author: Guillaume Tucker - -apiVersion: v1 -kind: Pod +apiVersion: apps/v1 +kind: Deployment metadata: name: scheduler-k8s - namespace: kernelci-pipeline + namespace: kernelci-pipeline-testns spec: - containers: - - name: scheduler - image: kernelci/pipeline - imagePullPolicy: Always - command: - - ./src/scheduler.py - - --settings=/home/kernelci/secrets/kernelci.toml - - loop - - --runtimes=k8s-gke-eu-west4 - env: - - name: KCI_API_TOKEN - valueFrom: - secretKeyRef: - name: kernelci-api-token - key: token - volumeMounts: - - name: secrets - mountPath: /home/kernelci/secrets - - name: secrets - mountPath: /home/kernelci/.kube - subPath: k8s-credentials/.kube - - name: secrets - mountPath: /home/kernelci/.config/gcloud - subPath: k8s-credentials/.config/gcloud - - name: secrets - mountPath: /home/kernelci/.azure - subPath: k8s-credentials/.azure - initContainers: - - name: settings - image: kernelci/pipeline - imagePullPolicy: Always - env: - - name: AZURE_FILES_TOKEN - valueFrom: - secretKeyRef: - name: azure-files-token - key: token - volumeMounts: - - name: secrets - mountPath: /tmp/secrets - command: - - /bin/bash - - -e - - -c - - "\ -cp /home/kernelci/pipeline/kube/aks/kernelci.toml /tmp/secrets/; \ -echo -e \"\ -\\n\ -[storage.early-access-azure]\\n\ -storage_cred = \\\"$AZURE_FILES_TOKEN\\\"\ -\" >> /tmp/secrets/kernelci.toml;" - - name: credentials - image: kernelci/pipeline - imagePullPolicy: Always - volumeMounts: - - name: secrets - mountPath: /tmp/secrets - - name: credentials - mountPath: /tmp/credentials - command: - - tar - - xzf - - /tmp/credentials/k8s-credentials.tar.gz - - -C - - /tmp/secrets - volumes: - - name: secrets - emptyDir: {} - - name: credentials - secret: - secretName: k8s-credentials + replicas: 1 + selector: + matchLabels: + app: scheduler-k8s + template: + metadata: + labels: + app: scheduler-k8s + spec: + containers: + - name: scheduler + image: kernelci/pipeline + imagePullPolicy: Always + command: + - ./src/scheduler.py + - --settings=/home/kernelci/secrets/kernelci-secrets.toml + - loop + - --runtimes=k8s-gke-eu-west4 + env: + - name: KCI_API_TOKEN + valueFrom: + secretKeyRef: + name: kernelci-api-token + key: token + volumeMounts: + - name: secrets + mountPath: /home/kernelci/secrets + volumes: + - name: secrets + secret: + secretName: kernelci-secrets diff --git a/kube/aks/scheduler-lava.yaml b/kube/aks/scheduler-lava.yaml index cacd8f958..12ae1a96e 100644 --- a/kube/aks/scheduler-lava.yaml +++ b/kube/aks/scheduler-lava.yaml @@ -1,59 +1,37 @@ -# SPDX-License-Identifier: LGPL-2.1-or-later -# -# Copyright (C) 2023 Collabora Limited -# Author: Guillaume Tucker - -apiVersion: v1 -kind: Pod +apiVersion: apps/v1 +kind: Deployment metadata: name: scheduler-lava - namespace: kernelci-pipeline + namespace: kernelci-pipeline-testns spec: - containers: - - name: scheduler - image: kernelci/pipeline - imagePullPolicy: Always - command: - - ./src/scheduler.py - - --settings=/home/kernelci/secrets/kernelci.toml - - loop - # Note: This sould be lava-collabora but the callback token name is - # different depending on the API instance (staging vs early-access). So - # for now we have 2 configs for the same runtime. - - --runtimes=lava-collabora-early-access - env: - - name: KCI_API_TOKEN - valueFrom: - secretKeyRef: - name: kernelci-api-token - key: token - volumeMounts: - - name: secrets - mountPath: /home/kernelci/secrets - initContainers: - - name: settings - image: kernelci/pipeline - imagePullPolicy: Always - env: - - name: LAVA_COLLABORA_TOKEN - valueFrom: - secretKeyRef: - name: lava-collabora-token - key: token - volumeMounts: - - name: secrets - mountPath: /tmp/secrets - command: - - /bin/bash - - -e - - -c - - "\ -cp /home/kernelci/pipeline/kube/aks/kernelci.toml /tmp/secrets/; \ -echo -e \"\ -\\n\ -[runtime.lava-collabora-early-access]\\n\ -runtime_token = \\\"$LAVA_COLLABORA_TOKEN\\\"\ -\" >> /tmp/secrets/kernelci.toml;" - volumes: - - name: secrets - emptyDir: {} + replicas: 1 + selector: + matchLabels: + app: scheduler-lava + template: + metadata: + labels: + app: scheduler-lava + spec: + containers: + - name: scheduler + image: kernelci/pipeline + imagePullPolicy: Always + command: + - ./src/scheduler.py + - --settings=/home/kernelci/secrets/kernelci-secrets.toml + - loop + - --runtimes=lava-collabora-early-access + env: + - name: KCI_API_TOKEN + valueFrom: + secretKeyRef: + name: kernelci-api-token + key: token + volumeMounts: + - name: secrets + mountPath: /home/kernelci/secrets + volumes: + - name: secrets + secret: + secretName: kernelci-secrets diff --git a/kube/aks/tarball.yaml b/kube/aks/tarball.yaml index ab29c2f7f..fd18d58ba 100644 --- a/kube/aks/tarball.yaml +++ b/kube/aks/tarball.yaml @@ -1,93 +1,47 @@ -# SPDX-License-Identifier: LGPL-2.1-or-later -# -# Copyright (C) 2023 Collabora Limited -# Author: Guillaume Tucker - -apiVersion: v1 -kind: Pod +apiVersion: apps/v1 +kind: Deployment metadata: name: tarball - namespace: kernelci-pipeline + namespace: kernelci-pipeline-testns spec: - containers: - - name: tarball - image: kernelci/pipeline - imagePullPolicy: Always - resources: - requests: - memory: 1Gi - cpu: 500m - limits: - memory: 4Gi - cpu: 2 - command: - - ./src/tarball.py - - --settings=/home/kernelci/secrets/kernelci.toml - - run - env: - - name: KCI_API_TOKEN - valueFrom: - secretKeyRef: - name: kernelci-api-token - key: token - volumeMounts: - - name: secrets - mountPath: /home/kernelci/secrets - - name: src - mountPath: /home/kernelci/pipeline/data/src - initContainers: - - name: secrets - image: kernelci/pipeline - imagePullPolicy: Always - env: - - name: AZURE_FILES_TOKEN - valueFrom: - secretKeyRef: - name: azure-files-token - key: token - volumeMounts: - - name: secrets - mountPath: /tmp/secrets - command: - - /bin/bash - - -e - - -c - - "\ -cp /home/kernelci/pipeline/kube/aks/kernelci.toml /tmp/secrets/; \ -echo -e \"\ -\\n\ -[storage.early-access-azure]\\n\ -storage_cred = \\\"$AZURE_FILES_TOKEN\\\"\ -\" >> /tmp/secrets/kernelci.toml;" - # Until we have a mirror on persistent storage, pre-populate a linux kernel - # checkout with some amount of git history to speed things up a bit - # https://github.com/kernelci/kernelci-pipeline/issues/310 - - name: git-clone - image: kernelci/pipeline - imagePullPolicy: Always - volumeMounts: - - name: src - mountPath: /tmp/src - command: - - git - - clone - - --depth=100 - - https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git - - /tmp/src/linux - - name: git-tags - image: kernelci/pipeline - imagePullPolicy: Always - volumeMounts: - - name: src - mountPath: /tmp/src - workingDir: /tmp/src/linux - command: - - git - - fetch - - --tags - - origin - volumes: - - name: src - emptyDir: {} - - name: secrets - emptyDir: {} + replicas: 1 + selector: + matchLabels: + app: tarball + template: + metadata: + labels: + app: tarball + spec: + containers: + - name: tarball + image: kernelci/pipeline + imagePullPolicy: Always + resources: + requests: + memory: 1Gi + cpu: 500m + limits: + memory: 4Gi + cpu: 2 + command: + - ./src/tarball.py + - --settings=/home/kernelci/secrets/kernelci.toml + - run + env: + - name: KCI_API_TOKEN + valueFrom: + secretKeyRef: + name: kernelci-api-token + key: token + volumeMounts: + - name: secrets + mountPath: /home/kernelci/secrets + - name: src + mountPath: /home/kernelci/pipeline/data/src + volumes: + - name: secrets + secret: + secretName: kernelci-secrets + - name: src + emptyDir: {} diff --git a/kube/aks/timeout.yaml b/kube/aks/timeout.yaml deleted file mode 100644 index 14190ecd6..000000000 --- a/kube/aks/timeout.yaml +++ /dev/null @@ -1,70 +0,0 @@ -# SPDX-License-Identifier: LGPL-2.1-or-later -# -# Copyright (C) 2023 Collabora Limited -# Author: Guillaume Tucker - -apiVersion: v1 -kind: Pod -metadata: - name: timeout - namespace: kernelci-pipeline -spec: - containers: - - name: timeout - image: kernelci/pipeline - imagePullPolicy: Always - command: - - ./src/timeout.py - - --settings=/home/kernelci/pipeline/kube/aks/kernelci.toml - - run - - --mode=timeout - env: - - name: KCI_API_TOKEN - valueFrom: - secretKeyRef: - name: kernelci-api-token - key: token ---- -apiVersion: v1 -kind: Pod -metadata: - name: closing - namespace: kernelci-pipeline -spec: - containers: - - name: timeout - image: kernelci/pipeline - imagePullPolicy: Always - command: - - ./src/timeout.py - - --settings=/home/kernelci/pipeline/kube/aks/kernelci.toml - - run - - --mode=closing - env: - - name: KCI_API_TOKEN - valueFrom: - secretKeyRef: - name: kernelci-api-token - key: token ---- -apiVersion: v1 -kind: Pod -metadata: - name: holdoff - namespace: kernelci-pipeline -spec: - containers: - - name: timeout - image: kernelci/pipeline - imagePullPolicy: Always - command: - - ./src/timeout.py - - --settings=/home/kernelci/pipeline/kube/aks/kernelci.toml - - run - - --mode=holdoff - env: - - name: KCI_API_TOKEN - valueFrom: - secretKeyRef: - name: kernelci-api-token - key: token diff --git a/kube/aks/trigger.yaml b/kube/aks/trigger.yaml index e6ec6dbb6..0894803bd 100644 --- a/kube/aks/trigger.yaml +++ b/kube/aks/trigger.yaml @@ -1,25 +1,29 @@ -# SPDX-License-Identifier: LGPL-2.1-or-later -# -# Copyright (C) 2023 Collabora Limited -# Author: Guillaume Tucker - -apiVersion: v1 -kind: Pod +apiVersion: apps/v1 +kind: Deployment metadata: name: trigger - namespace: kernelci-pipeline + namespace: kernelci-pipeline-testns spec: - containers: - - name: trigger - image: kernelci/pipeline - imagePullPolicy: Always - command: - - ./src/trigger.py - - --settings=/home/kernelci/pipeline/kube/aks/kernelci.toml - - run - env: - - name: KCI_API_TOKEN - valueFrom: - secretKeyRef: - name: kernelci-api-token - key: token + replicas: 1 + selector: + matchLabels: + app: trigger + template: + metadata: + labels: + app: trigger + spec: + containers: + - name: trigger + image: kernelci/pipeline + imagePullPolicy: Always + command: + - ./src/trigger.py + - --settings=/home/kernelci/pipeline/kube/aks/kernelci.toml + - run + env: + - name: KCI_API_TOKEN + valueFrom: + secretKeyRef: + name: kernelci-api-token + key: token