From a29a9ac9d476fc6e4d25187167f2d7c4f0e9530b Mon Sep 17 00:00:00 2001 From: "dmitry.denisenko" Date: Sun, 26 Jan 2025 18:08:35 +0400 Subject: [PATCH] add d8 collect-debug-info --- .../collect-debug-info/collect-debug-info.go | 62 +++++ .../createtarball/createtarball.go | 257 ++++++++++++++++++ internal/platform/cmd/platform.go | 2 + 3 files changed, 321 insertions(+) create mode 100644 internal/platform/cmd/collect-debug-info/collect-debug-info.go create mode 100644 internal/platform/cmd/collect-debug-info/createtarball/createtarball.go diff --git a/internal/platform/cmd/collect-debug-info/collect-debug-info.go b/internal/platform/cmd/collect-debug-info/collect-debug-info.go new file mode 100644 index 00000000..3d2e18f1 --- /dev/null +++ b/internal/platform/cmd/collect-debug-info/collect-debug-info.go @@ -0,0 +1,62 @@ +/* +Copyright 2024 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package collect_debug_info + +import ( + "fmt" + "github.com/deckhouse/deckhouse-cli/internal/platform/cmd/collect-debug-info/createtarball" + "github.com/deckhouse/deckhouse-cli/internal/platform/cmd/edit/flags" + "github.com/deckhouse/deckhouse-cli/internal/utilk8s" + "github.com/spf13/cobra" + "k8s.io/kubectl/pkg/util/templates" +) + +var collectDebugInfoCmdLong = templates.LongDesc(` +Collect debug info from Deckhouse Kubernetes Platform. + +© Flant JSC 2025`) + +func NewCommand() *cobra.Command { + collectDebugInfoCmd := &cobra.Command{ + Use: "collect-debug-info", + Short: "Collect debug info.", + Long: collectDebugInfoCmdLong, + SilenceErrors: true, + SilenceUsage: true, + RunE: collectDebugInfo, + } + flags.AddFlags(collectDebugInfoCmd.Flags()) + return collectDebugInfoCmd +} + +func collectDebugInfo(cmd *cobra.Command, _ []string) error { + kubeconfigPath, err := cmd.Flags().GetString("kubeconfig") + if err != nil { + return fmt.Errorf("Failed to setup Kubernetes client: %w", err) + } + + config, kubeCl, err := utilk8s.SetupK8sClientSet(kubeconfigPath) + if err != nil { + return fmt.Errorf("Failed to setup Kubernetes client: %w", err) + } + + err = createtarball.Tarball(config, kubeCl) + if err != nil { + return fmt.Errorf("Error collecting debug info: %w", err) + } + return err +} diff --git a/internal/platform/cmd/collect-debug-info/createtarball/createtarball.go b/internal/platform/cmd/collect-debug-info/createtarball/createtarball.go new file mode 100644 index 00000000..dfe826fe --- /dev/null +++ b/internal/platform/cmd/collect-debug-info/createtarball/createtarball.go @@ -0,0 +1,257 @@ +package createtarball + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "fmt" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/remotecommand" + "os" + "strings" +) + +type Command struct { + Cmd string + Args []string + File string +} + +func Tarball(config *rest.Config, kubeCl *kubernetes.Clientset) error { + const ( + labelSelector = "leader=true" + namespace = "d8-system" + containerName = "deckhouse" + ) + + debugCommands := []Command{ + { + File: "queue.txt", + Cmd: "deckhouse-controller", + Args: []string{"queue", "list"}, + }, + { + File: "global-values.json", + Cmd: "bash", + Args: []string{"-c", `deckhouse-controller global values -o json | jq '.internal.modules.kubeRBACProxyCA = "REDACTED" | .modulesImages.registry.dockercfg = "REDACTED"'`}, + }, + { + File: "deckhouse-enabled-modules.json", + Cmd: "bash", + Args: []string{"-c", "kubectl get modules -o json | jq '.items[]'"}, + }, + { + File: "events.json", + Cmd: "kubectl", + Args: []string{"get", "events", "--sort-by=.metadata.creationTimestamp", "-A", "-o", "json"}, + }, + { + File: "d8-all.json", + Cmd: "bash", + Args: []string{"-c", `for ns in $(kubectl get ns -o go-template='{{range .items}}{{.metadata.name}}{{"\n"}}{{end}}{{"kube-system"}}' -l heritage=deckhouse); do kubectl -n $ns get all -o json; done | jq -s '[.[].items[]]'`}, + }, + { + File: "node-groups.json", + Cmd: "kubectl", + Args: []string{"get", "nodegroups", "-A", "-o", "json"}, + }, + { + File: "nodes.json", + Cmd: "kubectl", + Args: []string{"get", "nodes", "-A", "-o", "json"}, + }, + { + File: "machines.json", + Cmd: "kubectl", + Args: []string{"get", "machines.machine.sapcloud.io", "-A", "-o", "json"}, + }, + { + File: "instances.json", + Cmd: "kubectl", + Args: []string{"get", "instances.deckhouse.io", "-o", "json"}, + }, + { + File: "staticinstances.json", + Cmd: "kubectl", + Args: []string{"get", "staticinstances.deckhouse.io", "-o", "json"}, + }, + { + File: "deckhouse-version.json", + Cmd: "bash", + Args: []string{"-c", "jq -s add <(kubectl -n d8-system get deployment deckhouse -o json | jq -r '.metadata.annotations | {\"core.deckhouse.io/edition\",\"core.deckhouse.io/version\"}') <(kubectl -n d8-system get deployment deckhouse -o json | jq -r '.spec.template.spec.containers[] | select(.name == \"deckhouse\") | {image}')"}, + }, + { + File: "deckhouse-releases.json", + Cmd: "kubectl", + Args: []string{"get", "deckhousereleases", "-o", "json"}, + }, + { + File: "deckhouse-logs.json", + Cmd: "kubectl", + Args: []string{"logs", "deploy/deckhouse", "--tail", "3000"}, + }, + { + File: "mcm-logs.txt", + Cmd: "kubectl", + Args: []string{"-n", "d8-cloud-instance-manager", "logs", "-l", "app=machine-controller-manager", "--tail", "3000", "-c", "controller"}, + }, + { + File: "ccm-logs.txt", + Cmd: "kubectl", + Args: []string{"-n", "d8-cloud-provider", "logs", "-l", "app=cloud-controller-manager", "--tail", "3000"}, + }, + { + File: "cluster-autoscaler-logs.txt", + Cmd: "kubectl", + Args: []string{"-n", "d8-cloud-instance-manager", "logs", "-l", "app=cluster-autoscaler", "--tail", "3000", "-c", "cluster-autoscaler"}, + }, + { + File: "vpa-admission-controller-logs.txt", + Cmd: "kubectl", + Args: []string{"-n", "kube-system", "logs", "-l", "app=vpa-admission-controller", "--tail", "3000", "-c", "admission-controller"}, + }, + { + File: "vpa-recommender-logs.txt", + Cmd: "kubectl", + Args: []string{"-n", "kube-system", "logs", "-l", "app=vpa-recommender", "--tail", "3000", "-c", "recommender"}, + }, + { + File: "vpa-updater-logs.txt", + Cmd: "kubectl", + Args: []string{"-n", "kube-system", "logs", "-l", "app=vpa-updater", "--tail", "3000", "-c", "updater"}, + }, + { + File: "prometheus-logs.txt", + Cmd: "kubectl", + Args: []string{"-n", "d8-monitoring", "logs", "-l", "prometheus=main", "--tail", "3000", "-c", "prometheus"}, + }, + { + File: "terraform-check.json", + Cmd: "bash", + Args: []string{"-c", `kubectl exec deploy/terraform-state-exporter -- dhctl terraform check --logger-type json -o json | jq -c '.terraform_plan[]?.variables.providerClusterConfiguration.value.provider = "REDACTED"'`}, + }, + { + File: "alerts.json", + Cmd: "bash", + Args: []string{"-c", `kubectl get clusteralerts.deckhouse.io -o json | jq '.items[]'`}, + }, + { + File: "pods.txt", + Cmd: "bash", + Args: []string{"-c", `kubectl get pod -A -owide | grep -Pv '\s+([1-9]+[\d]*)\/\1\s+' | grep -v 'Completed\|Evicted' | grep -E "^(d8-|kube-system)"`}, + }, + { + File: "cluster-authorization-rules.json", + Cmd: "kubectl", + Args: []string{"get", "clusterauthorizationrules", "-o", "json"}, + }, + { + File: "authorization-rules.json", + Cmd: "kubectl", + Args: []string{"get", "authorizationrules", "-o", "json"}, + }, + { + File: "module-configs.json", + Cmd: "kubectl", + Args: []string{"get", "moduleconfig", "-o", "json"}, + }, + } + + podName, err := getDeckhousePod(kubeCl, namespace, labelSelector, containerName) + + var stdout, stderr bytes.Buffer + gzipWriter := gzip.NewWriter(os.Stdout) + defer gzipWriter.Close() + tarWriter := tar.NewWriter(gzipWriter) + defer tarWriter.Close() + + for _, cmd := range debugCommands { + fullCommand := append([]string{cmd.Cmd}, cmd.Args...) + executor, err := execInPod(config, kubeCl, fullCommand, podName, namespace, containerName) + if err = executor.StreamWithContext( + context.Background(), + remotecommand.StreamOptions{ + Stdout: &stdout, + Stderr: &stderr, + }); err != nil { + fmt.Fprintf(os.Stderr, strings.Join(fullCommand, " ")) + fmt.Fprintf(os.Stderr, stderr.String()) + } + + err = cmd.Writer(tarWriter, stdout.Bytes()) + if err != nil { + return fmt.Errorf("failed to update the %s", err) + } + stdout.Reset() + + } + return err +} + +func (c *Command) Writer(tarWriter *tar.Writer, fileContent []byte) error { + header := &tar.Header{ + Name: c.File, + Mode: 0o600, + Size: int64(len(fileContent)), + } + + if err := tarWriter.WriteHeader(header); err != nil { + return fmt.Errorf("write tar header: %v", err) + } + + if _, err := tarWriter.Write(fileContent); err != nil { + return fmt.Errorf("copy content: %v", err) + } + + return nil +} + +func getDeckhousePod(kubeCl *kubernetes.Clientset, namespace string, labelSelector string, containerName string) (string, error) { + pods, err := kubeCl.CoreV1().Pods(namespace).List(context.Background(), metav1.ListOptions{ + LabelSelector: labelSelector, + }) + if err != nil { + return "", fmt.Errorf("Error listing pods: %w", err) + } + + if len(pods.Items) == 0 { + return "", fmt.Errorf("No pods found with the label: %s", labelSelector) + } + + pod := pods.Items[0] + podName := pod.Name + return podName, nil +} + +func execInPod(config *rest.Config, kubeCl *kubernetes.Clientset, fullCommand []string, podName string, namespace string, containerName string) (remotecommand.Executor, error) { + scheme := runtime.NewScheme() + parameterCodec := runtime.NewParameterCodec(scheme) + if err := v1.AddToScheme(scheme); err != nil { + return nil, fmt.Errorf("Failed to create parameter codec: %w", err) + } + req := kubeCl.CoreV1().RESTClient(). + Post(). + Resource("pods"). + Name(podName). + Namespace(namespace). + SubResource("exec"). + VersionedParams(&v1.PodExecOptions{ + Command: fullCommand, + Container: containerName, + Stdin: false, + Stdout: true, + Stderr: true, + TTY: false, + }, parameterCodec) + + executor, err := remotecommand.NewSPDYExecutor(config, "POST", req.URL()) + if err != nil { + return nil, fmt.Errorf("Creating SPDY executor for Pod %s: %v", podName, err) + } + return executor, nil +} diff --git a/internal/platform/cmd/platform.go b/internal/platform/cmd/platform.go index 9706e44e..05e8a21c 100644 --- a/internal/platform/cmd/platform.go +++ b/internal/platform/cmd/platform.go @@ -20,6 +20,7 @@ import ( "github.com/spf13/cobra" "k8s.io/kubectl/pkg/util/templates" + collect_debug_info "github.com/deckhouse/deckhouse-cli/internal/platform/cmd/collect-debug-info" "github.com/deckhouse/deckhouse-cli/internal/platform/cmd/edit" "github.com/deckhouse/deckhouse-cli/internal/platform/flags" ) @@ -40,6 +41,7 @@ func NewCommand() *cobra.Command { platformCmd.AddCommand( edit.NewCommand(), + collect_debug_info.NewCommand(), ) flags.AddPersistentFlags(platformCmd)