From 9079a8832fccec75ee44f99efce2b4d0b507ecb7 Mon Sep 17 00:00:00 2001
From: Trekkie Coder <trekkie@netlox.io>
Date: Wed, 14 Jun 2023 15:18:22 +0900
Subject: [PATCH] Enhancements to cicd:Wait for cluster ready in cluster tests
 #87

---
 cicd/k3s-cilium-cluster/master.sh      |  1 +
 cicd/k3s-cilium-cluster/wait_ready.sh  | 37 ++++++++++++++++++++++++++
 cicd/k3s-cilium-cluster/worker.sh      |  1 +
 cicd/k3s-flannel-cluster/master.sh     |  1 +
 cicd/k3s-flannel-cluster/wait_ready.sh | 37 ++++++++++++++++++++++++++
 cicd/k3s-flannel-cluster/worker.sh     |  1 +
 6 files changed, 78 insertions(+)
 create mode 100755 cicd/k3s-cilium-cluster/wait_ready.sh
 create mode 100755 cicd/k3s-flannel-cluster/wait_ready.sh

diff --git a/cicd/k3s-cilium-cluster/master.sh b/cicd/k3s-cilium-cluster/master.sh
index dd1a67e3..332b8b5b 100644
--- a/cicd/k3s-cilium-cluster/master.sh
+++ b/cicd/k3s-cilium-cluster/master.sh
@@ -23,3 +23,4 @@ sudo cp /var/lib/rancher/k3s/server/node-token /vagrant/node-token
 sudo cp /etc/rancher/k3s/k3s.yaml /vagrant/k3s.yaml
 sudo sed -i -e "s/127.0.0.1/${MASTER_IP}/g" /vagrant/k3s.yaml
 sudo kubectl apply -f /vagrant/kube-loxilb.yml
+/vagrant/wait_ready.sh
diff --git a/cicd/k3s-cilium-cluster/wait_ready.sh b/cicd/k3s-cilium-cluster/wait_ready.sh
new file mode 100755
index 00000000..5ff06e37
--- /dev/null
+++ b/cicd/k3s-cilium-cluster/wait_ready.sh
@@ -0,0 +1,37 @@
+#!/bin/bash
+
+function wait_cluster_ready {
+    Res=$(sudo kubectl get pods -A |
+    while IFS= read -r line; do
+        if [[ "$line" != *"Running"* && "$line" != *"READY"* ]]; then
+            echo "not ready"
+            return
+        fi
+    done)
+    if [[ $Res == *"not ready"* ]]; then
+        return 1
+    fi
+    return 0
+}
+
+function wait_cluster_ready_full {
+  i=1
+  nr=0
+  for ((;;)) do
+    wait_cluster_ready
+    nr=$?
+    if [[ $nr == 0 ]]; then
+        echo "Cluster is ready"
+        break
+    fi
+    i=$(( $i + 1 ))
+    if [[ $i -ge 40 ]]; then
+        echo "Cluster is not ready.Giving up"
+        exit 1
+    fi
+    echo "Cluster is not ready...."
+    sleep 10
+  done
+}
+
+wait_cluster_ready_full
diff --git a/cicd/k3s-cilium-cluster/worker.sh b/cicd/k3s-cilium-cluster/worker.sh
index ac796e11..be91e0e1 100644
--- a/cicd/k3s-cilium-cluster/worker.sh
+++ b/cicd/k3s-cilium-cluster/worker.sh
@@ -8,3 +8,4 @@ curl -sfL https://get.k3s.io | K3S_URL="https://${MASTER_ADDR}:6443" K3S_TOKEN="
 sudo kubectl apply -f /vagrant/nginx.yml
 sudo kubectl apply -f /vagrant/udp.yml
 #sudo kubectl apply -f /vagrant/sctp.yml
+/vagrant/wait_ready.sh
diff --git a/cicd/k3s-flannel-cluster/master.sh b/cicd/k3s-flannel-cluster/master.sh
index c048128f..d88fbcd4 100644
--- a/cicd/k3s-flannel-cluster/master.sh
+++ b/cicd/k3s-flannel-cluster/master.sh
@@ -9,3 +9,4 @@ sudo cp /var/lib/rancher/k3s/server/node-token /vagrant/node-token
 sudo cp /etc/rancher/k3s/k3s.yaml /vagrant/k3s.yaml
 sudo sed -i -e "s/127.0.0.1/${MASTER_IP}/g" /vagrant/k3s.yaml
 sudo kubectl apply -f /vagrant/kube-loxilb.yml
+/vagrant/wait_ready.sh
diff --git a/cicd/k3s-flannel-cluster/wait_ready.sh b/cicd/k3s-flannel-cluster/wait_ready.sh
new file mode 100755
index 00000000..5ff06e37
--- /dev/null
+++ b/cicd/k3s-flannel-cluster/wait_ready.sh
@@ -0,0 +1,37 @@
+#!/bin/bash
+
+function wait_cluster_ready {
+    Res=$(sudo kubectl get pods -A |
+    while IFS= read -r line; do
+        if [[ "$line" != *"Running"* && "$line" != *"READY"* ]]; then
+            echo "not ready"
+            return
+        fi
+    done)
+    if [[ $Res == *"not ready"* ]]; then
+        return 1
+    fi
+    return 0
+}
+
+function wait_cluster_ready_full {
+  i=1
+  nr=0
+  for ((;;)) do
+    wait_cluster_ready
+    nr=$?
+    if [[ $nr == 0 ]]; then
+        echo "Cluster is ready"
+        break
+    fi
+    i=$(( $i + 1 ))
+    if [[ $i -ge 40 ]]; then
+        echo "Cluster is not ready.Giving up"
+        exit 1
+    fi
+    echo "Cluster is not ready...."
+    sleep 10
+  done
+}
+
+wait_cluster_ready_full
diff --git a/cicd/k3s-flannel-cluster/worker.sh b/cicd/k3s-flannel-cluster/worker.sh
index 9c9fa137..84986cc0 100644
--- a/cicd/k3s-flannel-cluster/worker.sh
+++ b/cicd/k3s-flannel-cluster/worker.sh
@@ -8,3 +8,4 @@ curl -sfL https://get.k3s.io | K3S_URL="https://${MASTER_ADDR}:6443" K3S_TOKEN="
 sudo kubectl apply -f /vagrant/nginx.yml
 sudo kubectl apply -f /vagrant/udp.yml
 sudo kubectl apply -f /vagrant/sctp.yml
+/vagrant/wait_ready.sh