From 9c695a306f3af5885bfd32712f9f9f02e426a0c6 Mon Sep 17 00:00:00 2001 From: Salah Aldeen Al Saleh Date: Fri, 31 Jan 2025 07:39:18 -0800 Subject: [PATCH] handle panics --- operator/pkg/cli/upgrade_job.go | 17 ++++------------- pkg/addons2/upgrade.go | 9 ++++++--- pkg/addons2/util.go | 11 ----------- pkg/extensions/upgrade.go | 9 ++++++--- pkg/extensions/util.go | 11 ----------- pkg/helpers/string.go | 11 +++++++++++ 6 files changed, 27 insertions(+), 41 deletions(-) diff --git a/operator/pkg/cli/upgrade_job.go b/operator/pkg/cli/upgrade_job.go index 636edceb1..7d1c4bebc 100644 --- a/operator/pkg/cli/upgrade_job.go +++ b/operator/pkg/cli/upgrade_job.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "os" + "runtime/debug" "time" "github.com/google/uuid" @@ -11,6 +12,7 @@ import ( "github.com/replicatedhq/embedded-cluster/operator/pkg/cli/migratev2" "github.com/replicatedhq/embedded-cluster/operator/pkg/k8sutil" "github.com/replicatedhq/embedded-cluster/operator/pkg/upgrade" + "github.com/replicatedhq/embedded-cluster/pkg/helpers" "github.com/replicatedhq/embedded-cluster/pkg/metrics" "github.com/replicatedhq/embedded-cluster/pkg/runtimeconfig" "github.com/replicatedhq/embedded-cluster/pkg/versions" @@ -108,7 +110,7 @@ func performUpgrade(ctx context.Context, kcli client.Client, in *ecv1beta1.Insta func attemptUpgrade(ctx context.Context, kcli client.Client, in *ecv1beta1.Installation) (finalErr error) { defer func() { if r := recover(); r != nil { - finalErr = fmt.Errorf("upgrade recovered from panic: %v", r) + finalErr = fmt.Errorf("upgrade recovered from panic: %v: %s", r, string(debug.Stack())) } }() @@ -134,7 +136,7 @@ func maybeMarkAsFailed(ctx context.Context, kcli client.Client, in *ecv1beta1.In if !lastAttempt { return nil } - if err := k8sutil.SetInstallationState(ctx, kcli, in.Name, ecv1beta1.InstallationStateFailed, cleanErrorMessage(err)); err != nil { + if err := k8sutil.SetInstallationState(ctx, kcli, in.Name, ecv1beta1.InstallationStateFailed, helpers.CleanErrorMessage(err)); err != nil { return fmt.Errorf("set installation state: %w", err) } return nil @@ -153,14 +155,3 @@ func isLastAttempt(ctx context.Context, kcli client.Client) (bool, error) { return job.Status.Failed >= *job.Spec.BackoffLimit, nil } - -func cleanErrorMessage(err error) string { - if err == nil { - return "" - } - msg := err.Error() - if len(msg) > 1024 { - msg = msg[:1024] - } - return msg -} diff --git a/pkg/addons2/upgrade.go b/pkg/addons2/upgrade.go index fe4bea581..9198f3542 100644 --- a/pkg/addons2/upgrade.go +++ b/pkg/addons2/upgrade.go @@ -3,6 +3,7 @@ package addons2 import ( "context" "fmt" + "runtime/debug" "github.com/pkg/errors" ecv1beta1 "github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1" @@ -16,6 +17,7 @@ import ( "github.com/replicatedhq/embedded-cluster/pkg/addons2/types" "github.com/replicatedhq/embedded-cluster/pkg/addons2/velero" "github.com/replicatedhq/embedded-cluster/pkg/helm" + "github.com/replicatedhq/embedded-cluster/pkg/helpers" "github.com/replicatedhq/embedded-cluster/pkg/kubeutils" "github.com/replicatedhq/embedded-cluster/pkg/runtimeconfig" "github.com/replicatedhq/embedded-cluster/pkg/versions" @@ -123,9 +125,10 @@ func upgradeAddOn(ctx context.Context, hcli *helm.Helm, kcli client.Client, in * return errors.Wrap(err, "set condition status") } - // TODO (@salah): handle panics - defer func() { + if r := recover(); r != nil { + finalErr = fmt.Errorf("upgrading %s recovered from panic: %v: %s", addon.Name(), r, string(debug.Stack())) + } if finalErr == nil { // mark as processed successfully if err := k8sutil.SetConditionStatus(ctx, kcli, in, metav1.Condition{ @@ -141,7 +144,7 @@ func upgradeAddOn(ctx context.Context, hcli *helm.Helm, kcli client.Client, in * Type: conditionName(addon), Status: metav1.ConditionFalse, Reason: "UpgradeFailed", - Message: cleanErrorMessage(finalErr), + Message: helpers.CleanErrorMessage(finalErr), }); err != nil { fmt.Printf("failed to set condition status: %v", err) } diff --git a/pkg/addons2/util.go b/pkg/addons2/util.go index ade00eaeb..d581dedf3 100644 --- a/pkg/addons2/util.go +++ b/pkg/addons2/util.go @@ -45,14 +45,3 @@ func operatorImages(images []string) (string, string, string, error) { return repo, tag, ecUtilsImage, nil } - -func cleanErrorMessage(err error) string { - if err == nil { - return "" - } - msg := err.Error() - if len(msg) > 1024 { - msg = msg[:1024] - } - return msg -} diff --git a/pkg/extensions/upgrade.go b/pkg/extensions/upgrade.go index a5c7cb0ef..dff1c60a1 100644 --- a/pkg/extensions/upgrade.go +++ b/pkg/extensions/upgrade.go @@ -3,11 +3,13 @@ package extensions import ( "context" "fmt" + "runtime/debug" "github.com/pkg/errors" ecv1beta1 "github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1" "github.com/replicatedhq/embedded-cluster/operator/pkg/k8sutil" "github.com/replicatedhq/embedded-cluster/pkg/helm" + "github.com/replicatedhq/embedded-cluster/pkg/helpers" "github.com/replicatedhq/embedded-cluster/pkg/runtimeconfig" "github.com/replicatedhq/embedded-cluster/pkg/versions" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -99,9 +101,10 @@ func handleExtension(ctx context.Context, hcli *helm.Helm, kcli client.Client, i return errors.Wrap(err, "set condition status") } - // TODO (@salah): handle panics - defer func() { + if r := recover(); r != nil { + finalErr = fmt.Errorf("%s %s recovered from panic: %v: %s", actionIng, ext.Name, r, string(debug.Stack())) + } if finalErr == nil { // mark as processed successfully if err := k8sutil.SetConditionStatus(ctx, kcli, in, metav1.Condition{ @@ -117,7 +120,7 @@ func handleExtension(ctx context.Context, hcli *helm.Helm, kcli client.Client, i Type: conditionName(ext), Status: metav1.ConditionFalse, Reason: action + "Failed", - Message: cleanErrorMessage(finalErr), + Message: helpers.CleanErrorMessage(finalErr), }); err != nil { fmt.Printf("failed to set condition status: %v", err) } diff --git a/pkg/extensions/util.go b/pkg/extensions/util.go index 61ef513c2..74dfd75eb 100644 --- a/pkg/extensions/util.go +++ b/pkg/extensions/util.go @@ -159,14 +159,3 @@ func diffExtensions(oldExts, newExts ecv1beta1.Extensions) DiffResult { func conditionName(ext ecv1beta1.Chart) string { return fmt.Sprintf("%s-%s", ext.TargetNS, ext.Name) } - -func cleanErrorMessage(err error) string { - if err == nil { - return "" - } - msg := err.Error() - if len(msg) > 1024 { - msg = msg[:1024] - } - return msg -} diff --git a/pkg/helpers/string.go b/pkg/helpers/string.go index 5431255aa..b81285bfc 100644 --- a/pkg/helpers/string.go +++ b/pkg/helpers/string.go @@ -24,3 +24,14 @@ func UniqueStringSlice(input []string) []string { } return unique } + +func CleanErrorMessage(err error) string { + if err == nil { + return "" + } + msg := err.Error() + if len(msg) > 1024 { + msg = msg[:1024] + } + return msg +}