EVEREST-1349 Fix DBs stuck in the "Deleting" state (#491)

controllers/common/consts.go

@@ -63,6 +63,10 @@ const (
 	// EverestSecretsPrefix is the prefix for secrets created by Everest.
 	EverestSecretsPrefix = "everest-secrets-"
 
+	// DBBackupCleanupFinalizer is the finalizer for cleaning up DatabaseClusterBackup.
+	// Deprecated: We keep this for backward compatibility.
+	DBBackupCleanupFinalizer = "everest.percona.com/dbb-cleanup"
+
 	// UpstreamClusterCleanupFinalizer is the finalizer for cleaning up the upstream cluster.
 	UpstreamClusterCleanupFinalizer = "everest.percona.com/upstream-cluster-cleanup"
 

controllers/common/helper.go

@@ -621,6 +621,51 @@ func GetRepoNameByBackupStorage(
 	return ""
 }
 
+// HandleDBBackupsCleanup handles the cleanup of the dbbackup objects.
+// Returns true if cleanup is complete.
+func HandleDBBackupsCleanup(
+	ctx context.Context,
+	c client.Client,
+	database *everestv1alpha1.DatabaseCluster,
+) (bool, error) {
+	if controllerutil.ContainsFinalizer(database, DBBackupCleanupFinalizer) {
+		if done, err := deleteBackupsForDatabase(ctx, c, database.GetName(), database.GetNamespace()); err != nil {
+			return false, err
+		} else if !done {
+			return false, nil
+		}
+		controllerutil.RemoveFinalizer(database, DBBackupCleanupFinalizer)
+		return true, c.Update(ctx, database)
+	}
+	return true, nil
+}
+
+// Delete all dbbackups for the given database.
+// Returns true if no dbbackups are found.
+func deleteBackupsForDatabase(
+	ctx context.Context,
+	c client.Client,
+	dbName, dbNs string,
+) (bool, error) {
+	backupList, err := ListDatabaseClusterBackups(ctx, c, dbName, dbNs)
+	if err != nil {
+		return false, err
+	}
+	if len(backupList.Items) == 0 {
+		return true, nil
+	}
+	for _, backup := range backupList.Items {
+		if !backup.GetDeletionTimestamp().IsZero() {
+			// Already deleting, continue to next.
+			continue
+		}
+		if err := c.Delete(ctx, &backup); err != nil {
+			return false, err
+		}
+	}
+	return false, nil
+}
+
 // HandleUpstreamClusterCleanup handles the cleanup of the psdmb objects.
 // Returns true if cleanup is complete.
 func HandleUpstreamClusterCleanup(

controllers/providers/pg/provider.go

@@ -134,6 +134,12 @@ func (p *Provider) Status(ctx context.Context) (everestv1alpha1.DatabaseClusterS
 
 // Cleanup runs the cleanup routines and returns true if the cleanup is done.
 func (p *Provider) Cleanup(ctx context.Context, database *everestv1alpha1.DatabaseCluster) (bool, error) {
+	// Even though we no longer set the DBBackupCleanupFinalizer, we still need
+	// to handle the cleanup to ensure backward compatibility.
+	done, err := common.HandleDBBackupsCleanup(ctx, p.C, database)
+	if err != nil || !done {
+		return done, err
+	}
 	return common.HandleUpstreamClusterCleanup(ctx, p.C, database, &pgv2.PerconaPGCluster{})
 }
 

controllers/providers/psmdb/provider.go

@@ -160,6 +160,12 @@ func (p *Provider) Cleanup(ctx context.Context, database *everestv1alpha1.Databa
 	if err != nil {
 		return false, err
 	}
+	// Even though we no longer set the DBBackupCleanupFinalizer, we still need
+	// to handle the cleanup to ensure backward compatibility.
+	done, err := common.HandleDBBackupsCleanup(ctx, p.C, database)
+	if err != nil || !done {
+		return done, err
+	}
 	return common.HandleUpstreamClusterCleanup(ctx, p.C, database, &psmdbv1.PerconaServerMongoDB{})
 }
 

controllers/providers/pxc/provider.go

@@ -230,6 +230,12 @@ func (p *Provider) Status(ctx context.Context) (everestv1alpha1.DatabaseClusterS
 
 // Cleanup runs the cleanup routines and returns true if the cleanup is done.
 func (p *Provider) Cleanup(ctx context.Context, database *everestv1alpha1.DatabaseCluster) (bool, error) {
+	// Even though we no longer set the DBBackupCleanupFinalizer, we still need
+	// to handle the cleanup to ensure backward compatibility.
+	done, err := common.HandleDBBackupsCleanup(ctx, p.C, database)
+	if err != nil || !done {
+		return done, err
+	}
 	return common.HandleUpstreamClusterCleanup(ctx, p.C, database, &pxcv1.PerconaXtraDBCluster{})
 }