diff --git a/internal/platform/cmd/collect-debug-info/collect-debug-info.go b/internal/platform/cmd/collect-debug-info/collect-debug-info.go new file mode 100644 index 00000000..a94c0b6f --- /dev/null +++ b/internal/platform/cmd/collect-debug-info/collect-debug-info.go @@ -0,0 +1,69 @@ +/* +Copyright 2024 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package collect_debug_info + +import ( + "fmt" + "github.com/deckhouse/deckhouse-cli/internal/platform/cmd/collect-debug-info/debugTar" + "github.com/deckhouse/deckhouse-cli/internal/utilk8s" + "github.com/spf13/cobra" + "golang.org/x/term" + "k8s.io/kubectl/pkg/util/templates" + "os" +) + +var collectDebugInfoCmdLong = templates.LongDesc(` +Collect debug info from Deckhouse Kubernetes Platform. + +© Flant JSC 2025`) + +func NewCommand() *cobra.Command { + collectDebugInfoCmd := &cobra.Command{ + Use: "collect-debug-info", + Short: "Collect debug info.", + Long: collectDebugInfoCmdLong, + SilenceErrors: true, + SilenceUsage: true, + PreRunE: func(cmd *cobra.Command, args []string) error { + var err error + if term.IsTerminal(int(os.Stdout.Fd())) { + return fmt.Errorf("Please provide output tar.gz to dump debug logs, ex. \"> dump-logs.tar.gz\"") + } + return err + }, + RunE: collectDebugInfo, + } + return collectDebugInfoCmd +} + +func collectDebugInfo(cmd *cobra.Command, _ []string) error { + kubeconfigPath, err := cmd.Flags().GetString("kubeconfig") + if err != nil { + return fmt.Errorf("Failed to setup Kubernetes client: %w", err) + } + + config, kubeCl, err := utilk8s.SetupK8sClientSet(kubeconfigPath) + if err != nil { + return fmt.Errorf("Failed to setup Kubernetes client: %w", err) + } + + err = debugTar.Tarball(config, kubeCl) + if err != nil { + return fmt.Errorf("Error collecting debug info: %w", err) + } + return err +} diff --git a/internal/platform/cmd/collect-debug-info/debugTar/debugTar.go b/internal/platform/cmd/collect-debug-info/debugTar/debugTar.go new file mode 100644 index 00000000..7c0debae --- /dev/null +++ b/internal/platform/cmd/collect-debug-info/debugTar/debugTar.go @@ -0,0 +1,215 @@ +package debugTar + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "fmt" + "github.com/deckhouse/deckhouse-cli/internal/platform/cmd/operatepod" + "golang.org/x/term" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/remotecommand" + "os" + "strings" +) + +type Command struct { + Cmd string + Args []string + File string +} + +func Tarball(config *rest.Config, kubeCl kubernetes.Interface) error { + const ( + labelSelector = "leader=true" + namespace = "d8-system" + containerName = "deckhouse" + ) + + debugCommands := []Command{ + { + File: "queue.txt", + Cmd: "deckhouse-controller", + Args: []string{"queue", "list"}, + }, + { + File: "global-values.json", + Cmd: "bash", + Args: []string{"-c", `deckhouse-controller global values -o json | jq '.internal.modules.kubeRBACProxyCA = "REDACTED" | .modulesImages.registry.dockercfg = "REDACTED"'`}, + }, + { + File: "deckhouse-enabled-modules.json", + Cmd: "bash", + Args: []string{"-c", "kubectl get modules -o json | jq '.items[]'"}, + }, + { + File: "events.json", + Cmd: "kubectl", + Args: []string{"get", "events", "--sort-by=.metadata.creationTimestamp", "-A", "-o", "json"}, + }, + { + File: "d8-all.json", + Cmd: "bash", + Args: []string{"-c", `for ns in $(kubectl get ns -o go-template='{{range .items}}{{.metadata.name}}{{"\n"}}{{end}}{{"kube-system"}}' -l heritage=deckhouse); do kubectl -n $ns get all -o json; done | jq -s '[.[].items[]]'`}, + }, + { + File: "node-groups.json", + Cmd: "kubectl", + Args: []string{"get", "nodegroups", "-A", "-o", "json"}, + }, + { + File: "nodes.json", + Cmd: "kubectl", + Args: []string{"get", "nodes", "-A", "-o", "json"}, + }, + { + File: "machines.json", + Cmd: "kubectl", + Args: []string{"get", "machines.machine.sapcloud.io", "-A", "-o", "json"}, + }, + { + File: "instances.json", + Cmd: "kubectl", + Args: []string{"get", "instances.deckhouse.io", "-o", "json"}, + }, + { + File: "staticinstances.json", + Cmd: "kubectl", + Args: []string{"get", "staticinstances.deckhouse.io", "-o", "json"}, + }, + { + File: "deckhouse-version.json", + Cmd: "bash", + Args: []string{"-c", "jq -s add <(kubectl -n d8-system get deployment deckhouse -o json | jq -r '.metadata.annotations | {\"core.deckhouse.io/edition\",\"core.deckhouse.io/version\"}') <(kubectl -n d8-system get deployment deckhouse -o json | jq -r '.spec.template.spec.containers[] | select(.name == \"deckhouse\") | {image}')"}, + }, + { + File: "deckhouse-releases.json", + Cmd: "kubectl", + Args: []string{"get", "deckhousereleases", "-o", "json"}, + }, + { + File: "deckhouse-logs.json", + Cmd: "kubectl", + Args: []string{"logs", "deploy/deckhouse", "--tail", "3000"}, + }, + { + File: "mcm-logs.txt", + Cmd: "kubectl", + Args: []string{"-n", "d8-cloud-instance-manager", "logs", "-l", "app=machine-controller-manager", "--tail", "3000", "-c", "controller"}, + }, + { + File: "ccm-logs.txt", + Cmd: "kubectl", + Args: []string{"-n", "d8-cloud-provider", "logs", "-l", "app=cloud-controller-manager", "--tail", "3000"}, + }, + { + File: "cluster-autoscaler-logs.txt", + Cmd: "kubectl", + Args: []string{"-n", "d8-cloud-instance-manager", "logs", "-l", "app=cluster-autoscaler", "--tail", "3000", "-c", "cluster-autoscaler"}, + }, + { + File: "vpa-admission-controller-logs.txt", + Cmd: "kubectl", + Args: []string{"-n", "kube-system", "logs", "-l", "app=vpa-admission-controller", "--tail", "3000", "-c", "admission-controller"}, + }, + { + File: "vpa-recommender-logs.txt", + Cmd: "kubectl", + Args: []string{"-n", "kube-system", "logs", "-l", "app=vpa-recommender", "--tail", "3000", "-c", "recommender"}, + }, + { + File: "vpa-updater-logs.txt", + Cmd: "kubectl", + Args: []string{"-n", "kube-system", "logs", "-l", "app=vpa-updater", "--tail", "3000", "-c", "updater"}, + }, + { + File: "prometheus-logs.txt", + Cmd: "kubectl", + Args: []string{"-n", "d8-monitoring", "logs", "-l", "prometheus=main", "--tail", "3000", "-c", "prometheus"}, + }, + { + File: "terraform-check.json", + Cmd: "bash", + Args: []string{"-c", `kubectl exec deploy/terraform-state-exporter -- dhctl terraform check --logger-type json -o json | jq -c '.terraform_plan[]?.variables.providerClusterConfiguration.value.provider = "REDACTED"'`}, + }, + { + File: "alerts.json", + Cmd: "bash", + Args: []string{"-c", `kubectl get clusteralerts.deckhouse.io -o json | jq '.items[]'`}, + }, + { + File: "pods.txt", + Cmd: "bash", + Args: []string{"-c", `kubectl get pod -A -owide | grep -Pv '\s+([1-9]+[\d]*)\/\1\s+' | grep -v 'Completed\|Evicted' | grep -E "^(d8-|kube-system)"`}, + }, + { + File: "cluster-authorization-rules.json", + Cmd: "kubectl", + Args: []string{"get", "clusterauthorizationrules", "-o", "json"}, + }, + { + File: "authorization-rules.json", + Cmd: "kubectl", + Args: []string{"get", "authorizationrules", "-o", "json"}, + }, + { + File: "module-configs.json", + Cmd: "kubectl", + Args: []string{"get", "moduleconfig", "-o", "json"}, + }, + } + + podName, err := operatepod.GetDeckhousePod(kubeCl, namespace, labelSelector) + + if term.IsTerminal(int(os.Stdout.Fd())) { + return fmt.Errorf("Please provide output tar.gz to dump debug logs, ex. \"> dump-logs.tar.gz\"") + } + + var stdout, stderr bytes.Buffer + gzipWriter := gzip.NewWriter(os.Stdout) + defer gzipWriter.Close() + tarWriter := tar.NewWriter(gzipWriter) + defer tarWriter.Close() + + for _, cmd := range debugCommands { + fullCommand := append([]string{cmd.Cmd}, cmd.Args...) + executor, err := operatepod.ExecInPod(config, kubeCl, fullCommand, podName, namespace, containerName) + if err = executor.StreamWithContext( + context.Background(), + remotecommand.StreamOptions{ + Stdout: &stdout, + Stderr: &stderr, + }); err != nil { + fmt.Fprintf(os.Stderr, strings.Join(fullCommand, " ")) + fmt.Fprintf(os.Stderr, stderr.String()) + } + + err = cmd.Writer(tarWriter, stdout.Bytes()) + if err != nil { + return fmt.Errorf("failed to update the %w", err) + } + stdout.Reset() + + } + return err +} + +func (c *Command) Writer(tarWriter *tar.Writer, fileContent []byte) error { + header := &tar.Header{ + Name: c.File, + Mode: 0o600, + Size: int64(len(fileContent)), + } + + if err := tarWriter.WriteHeader(header); err != nil { + return fmt.Errorf("write tar header: %v", err) + } + + if _, err := tarWriter.Write(fileContent); err != nil { + return fmt.Errorf("copy content: %v", err) + } + + return nil +} diff --git a/internal/platform/cmd/module/operatemodule/optionsmodule.go b/internal/platform/cmd/module/operatemodule/optionsmodule.go index d298347b..1f4602ec 100644 --- a/internal/platform/cmd/module/operatemodule/optionsmodule.go +++ b/internal/platform/cmd/module/operatemodule/optionsmodule.go @@ -4,9 +4,7 @@ import ( "bytes" "context" "fmt" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" + "github.com/deckhouse/deckhouse-cli/internal/platform/cmd/operatepod" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/remotecommand" @@ -26,8 +24,8 @@ func OptionsModule(config *rest.Config, kubeCl kubernetes.Interface, pathFromOpt fullEndpointUrl := fmt.Sprintf("%s://%s:%s/%s/%s", apiProtocol, apiEndpoint, apiPort, modulePath, pathFromOption) getApi := []string{"curl", fullEndpointUrl} - podName, err := getDeckhousePod(kubeCl, namespace, labelSelector) - executor, err := execInPod(config, kubeCl, getApi, podName, namespace, containerName) + podName, err := operatepod.GetDeckhousePod(kubeCl, namespace, labelSelector) + executor, err := operatepod.ExecInPod(config, kubeCl, getApi, podName, namespace, containerName) var stdout bytes.Buffer var stderr bytes.Buffer @@ -43,49 +41,3 @@ func OptionsModule(config *rest.Config, kubeCl kubernetes.Interface, pathFromOpt fmt.Printf("%s\n", stdout.String()) return err } - -func getDeckhousePod(kubeCl kubernetes.Interface, namespace string, labelSelector string) (string, error) { - pods, err := kubeCl.CoreV1().Pods(namespace).List(context.Background(), metav1.ListOptions{ - LabelSelector: labelSelector, - }) - if err != nil { - return "", fmt.Errorf("Error listing pods: %w", err) - } - - if len(pods.Items) == 0 { - return "", fmt.Errorf("No pods found with the label: %s", labelSelector) - } - - pod := pods.Items[0] - podName := pod.Name - return podName, nil -} - -func execInPod(config *rest.Config, kubeCl kubernetes.Interface, getApi []string, podName string, namespace string, containerName string) (remotecommand.Executor, error) { - scheme := runtime.NewScheme() - parameterCodec := runtime.NewParameterCodec(scheme) - if err := v1.AddToScheme(scheme); err != nil { - return nil, fmt.Errorf("Failed to create parameter codec: %w", err) - } - - req := kubeCl.CoreV1().RESTClient(). - Post(). - Resource("pods"). - Name(podName). - Namespace(namespace). - SubResource("exec"). - VersionedParams(&v1.PodExecOptions{ - Command: getApi, - Container: containerName, - Stdin: false, - Stdout: true, - Stderr: true, - TTY: false, - }, parameterCodec) - - executor, err := remotecommand.NewSPDYExecutor(config, "POST", req.URL()) - if err != nil { - return nil, fmt.Errorf("Creating SPDY executor for Pod %s: %v", podName, err) - } - return executor, nil -} diff --git a/internal/platform/cmd/operatepod/operatepod.go b/internal/platform/cmd/operatepod/operatepod.go new file mode 100644 index 00000000..48c12a77 --- /dev/null +++ b/internal/platform/cmd/operatepod/operatepod.go @@ -0,0 +1,58 @@ +package operatepod + +import ( + "context" + "fmt" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/remotecommand" +) + +func GetDeckhousePod(kubeCl kubernetes.Interface, namespace string, labelSelector string) (string, error) { + pods, err := kubeCl.CoreV1().Pods(namespace).List(context.Background(), metav1.ListOptions{ + LabelSelector: labelSelector, + }) + if err != nil { + return "", fmt.Errorf("Error listing pods: %w", err) + } + + if len(pods.Items) == 0 { + return "", fmt.Errorf("No pods found with the label: %s", labelSelector) + } + + pod := pods.Items[0] + podName := pod.Name + return podName, nil +} + +func ExecInPod(config *rest.Config, kubeCl kubernetes.Interface, cmdLine []string, podName string, namespace string, containerName string) (remotecommand.Executor, error) { + scheme := runtime.NewScheme() + parameterCodec := runtime.NewParameterCodec(scheme) + if err := v1.AddToScheme(scheme); err != nil { + return nil, fmt.Errorf("Failed to create parameter codec: %w", err) + } + + req := kubeCl.CoreV1().RESTClient(). + Post(). + Resource("pods"). + Name(podName). + Namespace(namespace). + SubResource("exec"). + VersionedParams(&v1.PodExecOptions{ + Command: cmdLine, + Container: containerName, + Stdin: false, + Stdout: true, + Stderr: true, + TTY: false, + }, parameterCodec) + + executor, err := remotecommand.NewSPDYExecutor(config, "POST", req.URL()) + if err != nil { + return nil, fmt.Errorf("Creating SPDY executor for Pod %s: %v", podName, err) + } + return executor, nil +} diff --git a/internal/platform/cmd/platform.go b/internal/platform/cmd/platform.go index 57410eb5..66447b09 100644 --- a/internal/platform/cmd/platform.go +++ b/internal/platform/cmd/platform.go @@ -20,6 +20,7 @@ import ( "github.com/spf13/cobra" "k8s.io/kubectl/pkg/util/templates" + collect_debug_info "github.com/deckhouse/deckhouse-cli/internal/platform/cmd/collect-debug-info" "github.com/deckhouse/deckhouse-cli/internal/platform/cmd/edit" "github.com/deckhouse/deckhouse-cli/internal/platform/cmd/module" "github.com/deckhouse/deckhouse-cli/internal/platform/flags" @@ -42,6 +43,7 @@ func NewCommand() *cobra.Command { platformCmd.AddCommand( edit.NewCommand(), module.NewCommand(), + collect_debug_info.NewCommand(), ) flags.AddPersistentFlags(platformCmd)