handle panics

operator/pkg/cli/upgrade_job.go

@@ -4,13 +4,15 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"runtime/debug"
 	"time"
 
 	"github.com/google/uuid"
 	ecv1beta1 "github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1"
 	"github.com/replicatedhq/embedded-cluster/operator/pkg/cli/migratev2"
 	"github.com/replicatedhq/embedded-cluster/operator/pkg/k8sutil"
 	"github.com/replicatedhq/embedded-cluster/operator/pkg/upgrade"
+	"github.com/replicatedhq/embedded-cluster/pkg/helpers"
 	"github.com/replicatedhq/embedded-cluster/pkg/metrics"
 	"github.com/replicatedhq/embedded-cluster/pkg/runtimeconfig"
 	"github.com/replicatedhq/embedded-cluster/pkg/versions"
@@ -108,7 +110,7 @@ func performUpgrade(ctx context.Context, kcli client.Client, in *ecv1beta1.Insta
 func attemptUpgrade(ctx context.Context, kcli client.Client, in *ecv1beta1.Installation) (finalErr error) {
 	defer func() {
 		if r := recover(); r != nil {
-			finalErr = fmt.Errorf("upgrade recovered from panic: %v", r)
+			finalErr = fmt.Errorf("upgrade recovered from panic: %v: %s", r, string(debug.Stack()))
 		}
 	}()
 
@@ -134,7 +136,7 @@ func maybeMarkAsFailed(ctx context.Context, kcli client.Client, in *ecv1beta1.In
 	if !lastAttempt {
 		return nil
 	}
-	if err := k8sutil.SetInstallationState(ctx, kcli, in.Name, ecv1beta1.InstallationStateFailed, cleanErrorMessage(err)); err != nil {
+	if err := k8sutil.SetInstallationState(ctx, kcli, in.Name, ecv1beta1.InstallationStateFailed, helpers.CleanErrorMessage(err)); err != nil {
 		return fmt.Errorf("set installation state: %w", err)
 	}
 	return nil
@@ -153,14 +155,3 @@ func isLastAttempt(ctx context.Context, kcli client.Client) (bool, error) {
 
 	return job.Status.Failed >= *job.Spec.BackoffLimit, nil
 }
-
-func cleanErrorMessage(err error) string {
-	if err == nil {
-		return ""
-	}
-	msg := err.Error()
-	if len(msg) > 1024 {
-		msg = msg[:1024]
-	}
-	return msg
-}

pkg/addons2/upgrade.go

@@ -3,6 +3,7 @@ package addons2
 import (
 	"context"
 	"fmt"
+	"runtime/debug"
 
 	"github.com/pkg/errors"
 	ecv1beta1 "github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1"
@@ -16,6 +17,7 @@ import (
 	"github.com/replicatedhq/embedded-cluster/pkg/addons2/types"
 	"github.com/replicatedhq/embedded-cluster/pkg/addons2/velero"
 	"github.com/replicatedhq/embedded-cluster/pkg/helm"
+	"github.com/replicatedhq/embedded-cluster/pkg/helpers"
 	"github.com/replicatedhq/embedded-cluster/pkg/kubeutils"
 	"github.com/replicatedhq/embedded-cluster/pkg/runtimeconfig"
 	"github.com/replicatedhq/embedded-cluster/pkg/versions"
@@ -123,9 +125,10 @@ func upgradeAddOn(ctx context.Context, hcli *helm.Helm, kcli client.Client, in *
 		return errors.Wrap(err, "set condition status")
 	}
 
-	// TODO (@salah): handle panics
-
 	defer func() {
+		if r := recover(); r != nil {
+			finalErr = fmt.Errorf("upgrading %s recovered from panic: %v: %s", addon.Name(), r, string(debug.Stack()))
+		}
 		if finalErr == nil {
 			// mark as processed successfully
 			if err := k8sutil.SetConditionStatus(ctx, kcli, in, metav1.Condition{
@@ -141,7 +144,7 @@ func upgradeAddOn(ctx context.Context, hcli *helm.Helm, kcli client.Client, in *
 				Type:    conditionName(addon),
 				Status:  metav1.ConditionFalse,
 				Reason:  "UpgradeFailed",
-				Message: cleanErrorMessage(finalErr),
+				Message: helpers.CleanErrorMessage(finalErr),
 			}); err != nil {
 				fmt.Printf("failed to set condition status: %v", err)
 			}

pkg/addons2/util.go

@@ -45,14 +45,3 @@ func operatorImages(images []string) (string, string, string, error) {
 
 	return repo, tag, ecUtilsImage, nil
 }
-
-func cleanErrorMessage(err error) string {
-	if err == nil {
-		return ""
-	}
-	msg := err.Error()
-	if len(msg) > 1024 {
-		msg = msg[:1024]
-	}
-	return msg
-}

pkg/extensions/upgrade.go

@@ -3,11 +3,13 @@ package extensions
 import (
 	"context"
 	"fmt"
+	"runtime/debug"
 
 	"github.com/pkg/errors"
 	ecv1beta1 "github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1"
 	"github.com/replicatedhq/embedded-cluster/operator/pkg/k8sutil"
 	"github.com/replicatedhq/embedded-cluster/pkg/helm"
+	"github.com/replicatedhq/embedded-cluster/pkg/helpers"
 	"github.com/replicatedhq/embedded-cluster/pkg/runtimeconfig"
 	"github.com/replicatedhq/embedded-cluster/pkg/versions"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -99,9 +101,10 @@ func handleExtension(ctx context.Context, hcli *helm.Helm, kcli client.Client, i
 		return errors.Wrap(err, "set condition status")
 	}
 
-	// TODO (@salah): handle panics
-
 	defer func() {
+		if r := recover(); r != nil {
+			finalErr = fmt.Errorf("%s %s recovered from panic: %v: %s", actionIng, ext.Name, r, string(debug.Stack()))
+		}
 		if finalErr == nil {
 			// mark as processed successfully
 			if err := k8sutil.SetConditionStatus(ctx, kcli, in, metav1.Condition{
@@ -117,7 +120,7 @@ func handleExtension(ctx context.Context, hcli *helm.Helm, kcli client.Client, i
 				Type:    conditionName(ext),
 				Status:  metav1.ConditionFalse,
 				Reason:  action + "Failed",
-				Message: cleanErrorMessage(finalErr),
+				Message: helpers.CleanErrorMessage(finalErr),
 			}); err != nil {
 				fmt.Printf("failed to set condition status: %v", err)
 			}

pkg/extensions/util.go

@@ -159,14 +159,3 @@ func diffExtensions(oldExts, newExts ecv1beta1.Extensions) DiffResult {
 func conditionName(ext ecv1beta1.Chart) string {
 	return fmt.Sprintf("%s-%s", ext.TargetNS, ext.Name)
 }
-
-func cleanErrorMessage(err error) string {
-	if err == nil {
-		return ""
-	}
-	msg := err.Error()
-	if len(msg) > 1024 {
-		msg = msg[:1024]
-	}
-	return msg
-}

pkg/helpers/string.go

@@ -24,3 +24,14 @@ func UniqueStringSlice(input []string) []string {
 	}
 	return unique
 }
+
+func CleanErrorMessage(err error) string {
+	if err == nil {
+		return ""
+	}
+	msg := err.Error()
+	if len(msg) > 1024 {
+		msg = msg[:1024]
+	}
+	return msg
+}