metrics: tracking workflows with waiting status (#41)

* metrics: tracking workflows with waiting status * metrics: tracking workflows with waiting status * fix unit tests Signed-off-by: ibraheem Al Saady <ibraheemalsaady@ibraheems-MacBook-Pro-2.local> Signed-off-by: ibraheem Al Saady <ibraheemalsaady@ibraheems-MacBook-Pro-2.local> Co-authored-by: ibraheem Al Saady <ibraheemalsaady@ibraheems-MacBook-Pro-2.local>
kuptan · Aug 14, 2022 · a9a0f2d · a9a0f2d
1 parent dd9dfb2
commit a9a0f2d
Show file tree

Hide file tree

Showing 4 changed files with 77 additions and 55 deletions.
diff --git a/config/samples/role-terraform-runner.yaml b/config/samples/role-terraform-runner.yaml
@@ -1,9 +1,3 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: terraform-runner
-  namespace: default
----
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:

diff --git a/controllers/terraform_controller_operation.go b/controllers/terraform_controller_operation.go
@@ -17,8 +17,13 @@ import (
 func (r *TerraformReconciler) updateRunStatus(ctx context.Context, run *v1alpha1.Terraform, status v1alpha1.TerraformRunStatus) {
 	run.Status.RunStatus = status
 
+	// set completion time of the run only if status is completed/failed
 	if status == v1alpha1.RunCompleted || status == v1alpha1.RunFailed {
 		run.Status.CompletionTime = time.Now().Format(time.UnixDate)
+	}
+
+	// record the status only if completed/failed/waiting
+	if status == v1alpha1.RunCompleted || status == v1alpha1.RunFailed || status == v1alpha1.RunWaitingForDependency {
 		r.MetricsRecorder.RecordStatus(run.Name, run.Namespace, status)
 	}
 

diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go
@@ -70,6 +70,10 @@ func (r *Recorder) RecordTotal(name string, namespace string) {
 func (r *Recorder) RecordStatus(name string, namespace string, status v1alpha1.TerraformRunStatus) {
 	var value float64
 
+	if status == v1alpha1.RunWaitingForDependency {
+		value = -1
+	}
+
 	if status == v1alpha1.RunFailed {
 		value = 1
 	}

diff --git a/internal/metrics/metrics_test.go b/internal/metrics/metrics_test.go
@@ -39,61 +39,80 @@ var _ = Describe("Metrics Recorder", func() {
 			Expect(metricFamilies[0].Metric[0].Counter).ToNot(BeNil())
 			Expect(metricFamilies[0].Metric[0].Counter.Value).To(Equal(&value))
 		})
+	})
+
+	Context("Recording Status", func() {
+		It("should record the waitingForDependency status", func() {
+			rec.RecordStatus(name, namespace, v1alpha1.RunWaitingForDependency)
+
+			var (
+				value      float64 = -1.0
+				metricName string  = "tfo_workflow_status"
+			)
 
-		Context("Recording Status", func() {
-			It("should record the completed status", func() {
-				rec.RecordStatus(name, namespace, v1alpha1.RunCompleted)
-
-				var (
-					value      float64 = 0.0
-					metricName string  = "tfo_workflow_status"
-				)
-
-				metricFamilies, err := reg.Gather()
-
-				Expect(err).ToNot(HaveOccurred())
-				Expect(metricFamilies).To(HaveLen(2))
-				Expect(metricFamilies[0].Name).To(Equal(&metricName))
-				Expect(metricFamilies[0].Metric).To(HaveLen(1))
-				Expect(metricFamilies[0].Metric[0].Gauge).ToNot(BeNil())
-				Expect(metricFamilies[0].Metric[0].Gauge.Value).To(Equal(&value))
-			})
-
-			It("should record the failed status", func() {
-				rec.RecordStatus(name+"failed", namespace, v1alpha1.RunFailed)
-
-				var (
-					value      float64 = 1.0
-					metricName string  = "tfo_workflow_status"
-				)
-
-				metricFamilies, err := reg.Gather()
-
-				Expect(err).ToNot(HaveOccurred())
-				Expect(metricFamilies).To(HaveLen(2))
-				Expect(metricFamilies[0].Name).To(Equal(&metricName))
-				Expect(metricFamilies[0].Metric).To(HaveLen(2))
-				Expect(metricFamilies[0].Metric[1].Gauge).ToNot(BeNil())
-				Expect(metricFamilies[0].Metric[1].Gauge.Value).To(Equal(&value))
-			})
+			metricFamilies, err := reg.Gather()
+
+			Expect(err).ToNot(HaveOccurred())
+			Expect(metricFamilies).To(HaveLen(2))
+			Expect(metricFamilies[0].Name).To(Equal(&metricName))
+			Expect(metricFamilies[0].Metric).To(HaveLen(1))
+			Expect(metricFamilies[0].Metric[0].Gauge).ToNot(BeNil())
+			Expect(metricFamilies[0].Metric[0].Gauge.Value).To(Equal(&value))
 		})
 
-		Context("Recording Duration", func() {
-			It("should record the duration metric", func() {
-				rec.RecordDuration(name, namespace, time.Now())
+		It("should record the failed status", func() {
+			rec.RecordStatus(name, namespace, v1alpha1.RunFailed)
 
-				var (
-					metricName string = "tfo_workflow_duration_seconds"
-				)
+			var (
+				value      float64 = 1.0
+				metricName string  = "tfo_workflow_status"
+			)
+
+			metricFamilies, err := reg.Gather()
+
+			Expect(err).ToNot(HaveOccurred())
+			Expect(metricFamilies).To(HaveLen(2))
+			Expect(metricFamilies[0].Name).To(Equal(&metricName))
+			Expect(metricFamilies[0].Metric).To(HaveLen(1))
+			Expect(metricFamilies[0].Metric[0].Gauge).ToNot(BeNil())
+			Expect(metricFamilies[0].Metric[0].Gauge.Value).To(Equal(&value))
+		})
 
-				metricFamilies, err := reg.Gather()
+		It("should record the completed status", func() {
+			rec.RecordStatus(name, namespace, v1alpha1.RunCompleted)
 
-				Expect(err).ToNot(HaveOccurred())
-				Expect(metricFamilies).To(HaveLen(3))
-				Expect(metricFamilies[0].Name).To(Equal(&metricName))
-				Expect(metricFamilies[0].Metric).To(HaveLen(1))
-				Expect(metricFamilies[0].Metric[0].Histogram).ToNot(BeNil())
-			})
+			var (
+				value      float64 = 0.0
+				metricName string  = "tfo_workflow_status"
+			)
+
+			metricFamilies, err := reg.Gather()
+
+			Expect(err).ToNot(HaveOccurred())
+			Expect(metricFamilies).To(HaveLen(2))
+			Expect(metricFamilies[0].Name).To(Equal(&metricName))
+			Expect(metricFamilies[0].Metric).To(HaveLen(1))
+			Expect(metricFamilies[0].Metric[0].Gauge).ToNot(BeNil())
+			Expect(metricFamilies[0].Metric[0].Gauge.Value).To(Equal(&value))
+		})
+
+	})
+
+	Context("Recording Duration", func() {
+		It("should record the duration metric", func() {
+			rec.RecordDuration(name, namespace, time.Now())
+
+			var (
+				metricName string = "tfo_workflow_duration_seconds"
+			)
+
+			metricFamilies, err := reg.Gather()
+
+			Expect(err).ToNot(HaveOccurred())
+			Expect(metricFamilies).To(HaveLen(3))
+			Expect(metricFamilies[0].Name).To(Equal(&metricName))
+			Expect(metricFamilies[0].Metric).To(HaveLen(1))
+			Expect(metricFamilies[0].Metric[0].Histogram).ToNot(BeNil())
 		})
 	})
 })