From 07031527161b3d3ad6ad33b2e8c2bf575529a7ce Mon Sep 17 00:00:00 2001 From: Mukundan Sundararajan Date: Mon, 28 Sep 2020 10:36:46 -0700 Subject: [PATCH] Add dashboard to status command. Fail gracefully on error. (#473) --- cmd/status.go | 11 ++- pkg/kubernetes/pods.go | 8 ++ pkg/kubernetes/pods_test.go | 47 +++++++++ pkg/kubernetes/status.go | 128 ++++++++++++++---------- pkg/kubernetes/status_test.go | 179 ++++++++++++++++++++++++++++++++++ 5 files changed, 320 insertions(+), 53 deletions(-) create mode 100644 pkg/kubernetes/pods_test.go create mode 100644 pkg/kubernetes/status_test.go diff --git a/cmd/status.go b/cmd/status.go index 11c9c8d2e..535994a9a 100644 --- a/cmd/status.go +++ b/cmd/status.go @@ -19,11 +19,20 @@ var StatusCmd = &cobra.Command{ Use: "status", Short: "Shows the Dapr system services (control plane) health status.", Run: func(cmd *cobra.Command, args []string) { - status, err := kubernetes.Status() + sc, err := kubernetes.NewStatusClient() if err != nil { print.FailureStatusEvent(os.Stdout, err.Error()) os.Exit(1) } + status, err := sc.Status() + if err != nil { + print.FailureStatusEvent(os.Stdout, err.Error()) + os.Exit(1) + } + if len(status) == 0 { + print.FailureStatusEvent(os.Stdout, "No status returned. Is Dapr initialized in your cluster?") + os.Exit(1) + } table, err := gocsv.MarshalString(status) if err != nil { print.FailureStatusEvent(os.Stdout, err.Error()) diff --git a/pkg/kubernetes/pods.go b/pkg/kubernetes/pods.go index 8e027158a..97a453b8d 100644 --- a/pkg/kubernetes/pods.go +++ b/pkg/kubernetes/pods.go @@ -9,6 +9,14 @@ import ( k8s "k8s.io/client-go/kubernetes" ) +func ListPodsInterface(client k8s.Interface, labelSelector map[string]string) (*core_v1.PodList, error) { + opts := v1.ListOptions{} + if labelSelector != nil { + opts.LabelSelector = labels.FormatLabels(labelSelector) + } + return client.CoreV1().Pods(v1.NamespaceAll).List(opts) +} + func ListPods(client *k8s.Clientset, namespace string, labelSelector map[string]string) (*core_v1.PodList, error) { opts := v1.ListOptions{} if labelSelector != nil { diff --git a/pkg/kubernetes/pods_test.go b/pkg/kubernetes/pods_test.go new file mode 100644 index 000000000..e4e0eac7c --- /dev/null +++ b/pkg/kubernetes/pods_test.go @@ -0,0 +1,47 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +// ------------------------------------------------------------ + +package kubernetes + +import ( + "testing" + + "github.com/stretchr/testify/assert" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" +) + +func TestListPodsInterface(t *testing.T) { + t.Run("empty list pods", func(t *testing.T) { + k8s := fake.NewSimpleClientset() + output, err := ListPodsInterface(k8s, map[string]string{ + "test": "test", + }) + assert.Nil(t, err, "unexpected error") + assert.NotNil(t, output, "Expected empty list") + assert.Equal(t, 0, len(output.Items), "Expected length 0") + }) + t.Run("one matching pod", func(t *testing.T) { + k8s := fake.NewSimpleClientset((&v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "test", + Annotations: map[string]string{}, + Labels: map[string]string{ + "test": "test", + }, + }, + })) + output, err := ListPodsInterface(k8s, map[string]string{ + "test": "test", + }) + assert.Nil(t, err, "unexpected error") + assert.NotNil(t, output, "Expected non empty list") + assert.Equal(t, 1, len(output.Items), "Expected length 0") + assert.Equal(t, "test", output.Items[0].Name, "expected name to match") + assert.Equal(t, "test", output.Items[0].Namespace, "expected namespace to match") + }) +} diff --git a/pkg/kubernetes/status.go b/pkg/kubernetes/status.go index fa6143553..49684fb6c 100644 --- a/pkg/kubernetes/status.go +++ b/pkg/kubernetes/status.go @@ -7,16 +7,22 @@ package kubernetes import ( "fmt" + "os" "strings" "sync" + "errors" + "github.com/dapr/cli/pkg/age" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/dapr/cli/pkg/print" + k8s "k8s.io/client-go/kubernetes" ) -var ( - controlPlaneLabels = []string{"dapr-operator", "dapr-sentry", "dapr-placement", "dapr-sidecar-injector"} -) +var controlPlaneLabels = []string{"dapr-operator", "dapr-sentry", "dapr-placement", "dapr-sidecar-injector", "dapr-dashboard"} + +type StatusClient struct { + client k8s.Interface +} // StatusOutput represents the status of a named Dapr resource. type StatusOutput struct { @@ -30,13 +36,23 @@ type StatusOutput struct { Created string `csv:"CREATED"` } -// List status for Dapr resources. -func Status() ([]StatusOutput, error) { - client, err := Client() +// Create a new k8s client for status commands +func NewStatusClient() (*StatusClient, error) { + clientset, err := Client() if err != nil { return nil, err } + return &StatusClient{ + client: clientset, + }, nil +} +// List status for Dapr resources. +func (s *StatusClient) Status() ([]StatusOutput, error) { + client := s.client + if client == nil { + return nil, errors.New("kubernetes client not initialized") + } var wg sync.WaitGroup wg.Add(len(controlPlaneLabels)) @@ -45,60 +61,68 @@ func Status() ([]StatusOutput, error) { for _, lbl := range controlPlaneLabels { go func(label string) { - p, err := ListPods(client, v1.NamespaceAll, map[string]string{ + defer wg.Done() + // Query all namespaces for Dapr pods. + p, err := ListPodsInterface(client, map[string]string{ "app": label, }) - if err == nil { - pod := p.Items[0] - replicas := len(p.Items) - image := pod.Spec.Containers[0].Image - namespace := pod.GetNamespace() - age := age.GetAge(pod.CreationTimestamp.Time) - created := pod.CreationTimestamp.Format("2006-01-02 15:04.05") - version := image[strings.IndexAny(image, ":")+1:] - status := "" - - // loop through all replicas and update to Running/Healthy status only if all instances are Running and Healthy - healthy := "False" - running := true - - for _, p := range p.Items { - if p.Status.ContainerStatuses[0].State.Waiting != nil { - status = fmt.Sprintf("Waiting (%s)", p.Status.ContainerStatuses[0].State.Waiting.Reason) - } else if pod.Status.ContainerStatuses[0].State.Terminated != nil { - status = "Terminated" - } - - if p.Status.ContainerStatuses[0].State.Running == nil { - running = false - break - } - - if p.Status.ContainerStatuses[0].Ready { - healthy = "True" - } + if err != nil { + print.WarningStatusEvent(os.Stdout, "Failed to get status for %s: %s", label, err.Error()) + return + } + + if len(p.Items) == 0 { + return + } + pod := p.Items[0] + replicas := len(p.Items) + image := pod.Spec.Containers[0].Image + namespace := pod.GetNamespace() + age := age.GetAge(pod.CreationTimestamp.Time) + created := pod.CreationTimestamp.Format("2006-01-02 15:04.05") + version := image[strings.IndexAny(image, ":")+1:] + status := "" + + // loop through all replicas and update to Running/Healthy status only if all instances are Running and Healthy + healthy := "False" + running := true + + for _, p := range p.Items { + if p.Status.ContainerStatuses[0].State.Waiting != nil { + status = fmt.Sprintf("Waiting (%s)", p.Status.ContainerStatuses[0].State.Waiting.Reason) + } else if pod.Status.ContainerStatuses[0].State.Terminated != nil { + status = "Terminated" } - if running { - status = "Running" + if p.Status.ContainerStatuses[0].State.Running == nil { + running = false + + break } - s := StatusOutput{ - Name: label, - Namespace: namespace, - Created: created, - Age: age, - Status: status, - Version: version, - Healthy: healthy, - Replicas: replicas, + if p.Status.ContainerStatuses[0].Ready { + healthy = "True" } + } + + if running { + status = "Running" + } - m.Lock() - statuses = append(statuses, s) - m.Unlock() + s := StatusOutput{ + Name: label, + Namespace: namespace, + Created: created, + Age: age, + Status: status, + Version: version, + Healthy: healthy, + Replicas: replicas, } - wg.Done() + + m.Lock() + statuses = append(statuses, s) + m.Unlock() }(lbl) } diff --git a/pkg/kubernetes/status_test.go b/pkg/kubernetes/status_test.go new file mode 100644 index 000000000..f3629d2ee --- /dev/null +++ b/pkg/kubernetes/status_test.go @@ -0,0 +1,179 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +// ------------------------------------------------------------ + +package kubernetes + +import ( + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/fake" +) + +func newTestSimpleK8s(objects ...runtime.Object) *StatusClient { + client := StatusClient{} + client.client = fake.NewSimpleClientset(objects...) + return &client +} + +func TestStatus(t *testing.T) { + t.Run("empty status. dapr not init", func(t *testing.T) { + k8s := newTestSimpleK8s() + status, err := k8s.Status() + if err != nil { + t.Fatalf("%s status should not raise an error", err.Error()) + } + assert.Equal(t, 0, len(status), "Expected status to be empty list") + }) + t.Run("one status waiting", func(t *testing.T) { + k8s := newTestSimpleK8s((&v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dapr-dashboard", + Namespace: "dapr-system", + Annotations: map[string]string{}, + Labels: map[string]string{ + "app": "dapr-dashboard", + }, + CreationTimestamp: metav1.Time{ + Time: time.Now(), + }, + }, + Status: v1.PodStatus{ + ContainerStatuses: []v1.ContainerStatus{ + { + State: v1.ContainerState{ + Waiting: &v1.ContainerStateWaiting{ + Reason: "test", + Message: "test", + }, + }, + }, + }, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Image: "dapr-dashboard:0.0.1", + }, + }, + }, + })) + status, err := k8s.Status() + assert.Nil(t, err, "status should not raise an error") + assert.Equal(t, 1, len(status), "Expected status to be empty list") + stat := status[0] + assert.Equal(t, "dapr-dashboard", stat.Name, "expected name to match") + assert.Equal(t, "dapr-system", stat.Namespace, "expected namespace to match") + assert.Equal(t, "0.0.1", stat.Version, "expected version to match") + assert.Equal(t, 1, stat.Replicas, "expected replicas to match") + assert.Equal(t, "False", stat.Healthy, "expected health to match") + assert.True(t, strings.HasPrefix(stat.Status, "Waiting"), "expected waiting status") + }) + t.Run("one status running", func(t *testing.T) { + testTime := time.Now() + k8s := newTestSimpleK8s((&v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dapr-dashboard", + Namespace: "dapr-system", + Annotations: map[string]string{}, + Labels: map[string]string{ + "app": "dapr-dashboard", + }, + CreationTimestamp: metav1.Time{ + Time: testTime.Add(time.Duration(-20) * time.Minute), + }, + }, + Status: v1.PodStatus{ + ContainerStatuses: []v1.ContainerStatus{ + { + State: v1.ContainerState{ + Running: &v1.ContainerStateRunning{ + StartedAt: metav1.Time{ + Time: testTime.Add(time.Duration(-19) * time.Minute), + }, + }, + }, + Ready: true, + }, + }, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Image: "dapr-dashboard:0.0.1", + }, + }, + }, + })) + status, err := k8s.Status() + assert.Nil(t, err, "status should not raise an error") + assert.Equal(t, 1, len(status), "Expected status to be empty list") + stat := status[0] + assert.Equal(t, "dapr-dashboard", stat.Name, "expected name to match") + assert.Equal(t, "dapr-system", stat.Namespace, "expected namespace to match") + assert.Equal(t, "20m", stat.Age, "expected age to match") + assert.Equal(t, "0.0.1", stat.Version, "expected version to match") + assert.Equal(t, 1, stat.Replicas, "expected replicas to match") + assert.Equal(t, "True", stat.Healthy, "expected health to match") + assert.Equal(t, stat.Status, "Running", "expected running status") + }) + t.Run("one status terminated", func(t *testing.T) { + testTime := time.Now() + k8s := newTestSimpleK8s((&v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dapr-dashboard", + Namespace: "dapr-system", + Annotations: map[string]string{}, + Labels: map[string]string{ + "app": "dapr-dashboard", + }, + CreationTimestamp: metav1.Time{ + Time: testTime.Add(time.Duration(-20) * time.Minute), + }, + }, + Status: v1.PodStatus{ + ContainerStatuses: []v1.ContainerStatus{ + { + State: v1.ContainerState{ + Terminated: &v1.ContainerStateTerminated{ + ExitCode: 1, + }, + }, + }, + }, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Image: "dapr-dashboard:0.0.1", + }, + }, + }, + })) + status, err := k8s.Status() + assert.Nil(t, err, "status should not raise an error") + assert.Equal(t, 1, len(status), "Expected status to be empty list") + stat := status[0] + assert.Equal(t, "dapr-dashboard", stat.Name, "expected name to match") + assert.Equal(t, "dapr-system", stat.Namespace, "expected namespace to match") + assert.Equal(t, "20m", stat.Age, "expected age to match") + assert.Equal(t, "0.0.1", stat.Version, "expected version to match") + assert.Equal(t, 1, stat.Replicas, "expected replicas to match") + assert.Equal(t, "False", stat.Healthy, "expected health to match") + assert.Equal(t, stat.Status, "Terminated", "expected terminated status") + }) + t.Run("one status empty client", func(t *testing.T) { + k8s := &StatusClient{} + status, err := k8s.Status() + assert.NotNil(t, err, "status should raise an error") + assert.Equal(t, "kubernetes client not initialized", err.Error(), "expected errors to match") + assert.Nil(t, status, "expected nil for status") + }) +}