Skip to content

Commit

Permalink
EVEREST-1427 Fix PG use the same bucket (#512)
Browse files Browse the repository at this point in the history
  • Loading branch information
@oksana-grishchenko
oksana-grishchenko authored Sep 9, 2024
1 parent c3af0d3 commit 0133462
Show file tree
Hide file tree
Showing 3 changed files with 262 additions and 185 deletions.
29 changes: 15 additions & 14 deletions controllers/providers/pg/applier.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -716,7 +716,7 @@ func (p *applier) createPGBackrestSecret(
func (p *applier) addBackupStoragesByRestores(
backupList *everestv1alpha1.DatabaseClusterBackupList,
restoreList *everestv1alpha1.DatabaseClusterRestoreList,
backupStorages map[string]everestv1alpha1.BackupStorageSpec,
backupStorages map[string]everestv1alpha1.BackupStorage,
backupStoragesSecrets map[string]*corev1.Secret,
) error {
ctx := p.ctx
Expand Down Expand Up @@ -763,7 +763,7 @@ func (p *applier) addBackupStoragesByRestores(
fmt.Errorf("failed to get backup storage secret %s", backupStorage.Spec.CredentialsSecretName))
}

backupStorages[backupStorage.Name] = backupStorage.Spec
backupStorages[backupStorage.Name] = *backupStorage
backupStoragesSecrets[backupStorage.Name] = backupStorageSecret

// XXX We need to add this restore's backup storage to the list of
Expand Down Expand Up @@ -792,7 +792,7 @@ func (p *applier) addBackupStoragesByRestores(
// Add backup storages used by backup schedules to the list.
func (p *applier) addBackupStoragesBySchedules(
backupSchedules []everestv1alpha1.BackupSchedule,
backupStorages map[string]everestv1alpha1.BackupStorageSpec,
backupStorages map[string]everestv1alpha1.BackupStorage,
backupStoragesSecrets map[string]*corev1.Secret,
) error {
ctx := p.ctx
Expand All @@ -816,7 +816,7 @@ func (p *applier) addBackupStoragesBySchedules(
return errors.Join(err, fmt.Errorf("failed to get backup storage secret %s", backupStorage.Spec.CredentialsSecretName))
}

backupStorages[backupStorage.Name] = backupStorage.Spec
backupStorages[backupStorage.Name] = *backupStorage
backupStoragesSecrets[backupStorage.Name] = backupStorageSecret
}
return nil
Expand All @@ -825,7 +825,7 @@ func (p *applier) addBackupStoragesBySchedules(
// Add backup storages used by on-demand backups to the list.
func (p *applier) addBackupStoragesByOnDemandBackups(
backupList *everestv1alpha1.DatabaseClusterBackupList,
backupStorages map[string]everestv1alpha1.BackupStorageSpec,
backupStorages map[string]everestv1alpha1.BackupStorage,
backupStoragesSecrets map[string]*corev1.Secret,
) error {
ctx := p.ctx
Expand All @@ -847,7 +847,7 @@ func (p *applier) addBackupStoragesByOnDemandBackups(
return fmt.Errorf("failed to get backup storage secret '%s': %w", backupStorage.Spec.CredentialsSecretName, err)
}

backupStorages[backupStorage.Name] = backupStorage.Spec
backupStorages[backupStorage.Name] = *backupStorage
backupStoragesSecrets[backupStorage.Name] = backupStorageSecret
}
return nil
Expand Down Expand Up @@ -890,7 +890,7 @@ func (p *applier) reconcilePGBackupsSpec() (pgv2.Backups, error) {
return pgv2.Backups{}, err
}

backupStorages := map[string]everestv1alpha1.BackupStorageSpec{}
backupStorages := map[string]everestv1alpha1.BackupStorage{}
backupStoragesSecrets := map[string]*corev1.Secret{}

// Add backup storages used by on-demand backups to the list
Expand Down Expand Up @@ -1001,7 +1001,7 @@ func reconcilePGBackRestRepos(
oldRepos []crunchyv1beta1.PGBackRestRepo,
backupSchedules []everestv1alpha1.BackupSchedule,
backups []everestv1alpha1.DatabaseClusterBackup,
backupStorages map[string]everestv1alpha1.BackupStorageSpec,
backupStorages map[string]everestv1alpha1.BackupStorage,
backupStoragesSecrets map[string]*corev1.Secret,
engineStorage everestv1alpha1.Storage,
db *everestv1alpha1.DatabaseCluster,
Expand Down Expand Up @@ -1053,16 +1053,17 @@ func reconcilePGBackRestRepos(
return newRepos, reposReconciler.pgBackRestGlobal, pgBackrestIniBytes, nil
}

func backupStorageNameFromRepo(backupStorages map[string]everestv1alpha1.BackupStorageSpec, repo crunchyv1beta1.PGBackRestRepo) string {
for name, spec := range backupStorages {
func backupStorageNameFromRepo(backupStorages map[string]everestv1alpha1.BackupStorage, repo crunchyv1beta1.PGBackRestRepo, dbNamespace string) string {
for name, bs := range backupStorages {
if repo.S3 != nil &&
repo.S3.Bucket == spec.Bucket &&
repo.S3.Region == spec.Region &&
repo.S3.Endpoint == spec.EndpointURL {
dbNamespace == bs.Namespace &&
repo.S3.Bucket == bs.Spec.Bucket &&
repo.S3.Region == bs.Spec.Region &&
repo.S3.Endpoint == bs.Spec.EndpointURL {
return name
}

if repo.Azure != nil && repo.Azure.Container == spec.Bucket {
if repo.Azure != nil && repo.Azure.Container == bs.Spec.Bucket {
return name
}
}
Expand Down
30 changes: 15 additions & 15 deletions controllers/providers/pg/pg_repos_reconciler.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,7 @@ func removeString(s []string, i int) []string {
// Update repos based on the existing backups.
func (p *pgReposReconciler) reconcileBackups(
backups []everestv1alpha1.DatabaseClusterBackup,
backupStorages map[string]everestv1alpha1.BackupStorageSpec,
backupStorages map[string]everestv1alpha1.BackupStorage,
backupStoragesSecrets map[string]*corev1.Secret,
db *everestv1alpha1.DatabaseCluster,
) error {
Expand Down Expand Up @@ -163,7 +163,7 @@ func (p *pgReposReconciler) reconcileBackups(
if !ok {
return fmt.Errorf("failed to cast repo %v", er.Value)
}
repoBackupStorageName := backupStorageNameFromRepo(backupStorages, repo)
repoBackupStorageName := backupStorageNameFromRepo(backupStorages, repo, db.Namespace)
if backup.Spec.BackupStorageName != repoBackupStorageName {
continue
}
Expand All @@ -182,15 +182,15 @@ func (p *pgReposReconciler) reconcileBackups(
}
}

repo, err := genPGBackrestRepo(repoName, backupStorage, nil)
repo, err := genPGBackrestRepo(repoName, backupStorage.Spec, nil)
if err != nil {
return err
}
p.reposReconciled.PushBack(repo)
// Keep track of backup storages which are already in use by a repo
p.backupStoragesInRepos[backup.Spec.BackupStorageName] = struct{}{}

p.addRepoToPGGlobal(backupStorage.VerifyTLS, repo.Name, backupStorages[repo.Name].ForcePathStyle, nil, db)
p.addRepoToPGGlobal(backupStorage.Spec.VerifyTLS, repo.Name, backupStorages[repo.Name].Spec.ForcePathStyle, nil, db)
err = updatePGIni(p.pgBackRestSecretIni, backupStoragesSecrets[backup.Spec.BackupStorageName], repo)
if err != nil {
return errors.Join(err, errors.New("failed to add backup storage credentials to PGBackrest secret data"))
Expand All @@ -202,7 +202,7 @@ func (p *pgReposReconciler) reconcileBackups(
// Update the schedules of the repos which already exist but have the wrong
// schedule and move them to the reconciled list.
func (p *pgReposReconciler) reconcileExistingSchedules(
backupStorages map[string]everestv1alpha1.BackupStorageSpec,
backupStorages map[string]everestv1alpha1.BackupStorage,
backupStoragesSecrets map[string]*corev1.Secret,
db *everestv1alpha1.DatabaseCluster,
) error {
Expand All @@ -226,7 +226,7 @@ func (p *pgReposReconciler) reconcileExistingSchedules(
if !ok {
return fmt.Errorf("failed to cast repo %v", er.Value)
}
repoBackupStorageName := backupStorageNameFromRepo(backupStorages, repo)
repoBackupStorageName := backupStorageNameFromRepo(backupStorages, repo, db.Namespace)
if backupSchedule.BackupStorageName != repoBackupStorageName ||
repo.BackupSchedules == nil ||
repo.BackupSchedules.Full == nil ||
Expand All @@ -243,7 +243,7 @@ func (p *pgReposReconciler) reconcileExistingSchedules(
// Keep track of backup storages which are already in use by a repo
p.backupStoragesInRepos[backupSchedule.BackupStorageName] = struct{}{}

p.addRepoToPGGlobal(backupStorages[repo.Name].VerifyTLS, repo.Name, backupStorages[repo.Name].ForcePathStyle, &backupSchedule.RetentionCopies, db)
p.addRepoToPGGlobal(backupStorages[repo.Name].Spec.VerifyTLS, repo.Name, backupStorages[repo.Name].Spec.ForcePathStyle, &backupSchedule.RetentionCopies, db)
err := updatePGIni(p.pgBackRestSecretIni, backupStoragesSecrets[backupSchedule.BackupStorageName], repo)
if err != nil {
return errors.Join(err, errors.New("failed to add backup storage credentials to PGBackrest secret data"))
Expand All @@ -256,7 +256,7 @@ func (p *pgReposReconciler) reconcileExistingSchedules(

// Update repos based on the schedules.
func (p *pgReposReconciler) reconcileRepos(
backupStorages map[string]everestv1alpha1.BackupStorageSpec,
backupStorages map[string]everestv1alpha1.BackupStorage,
backupStoragesSecrets map[string]*corev1.Secret,
db *everestv1alpha1.DatabaseCluster,
) error {
Expand All @@ -270,7 +270,7 @@ func (p *pgReposReconciler) reconcileRepos(
if !ok {
return fmt.Errorf("failed to cast repo %v", er.Value)
}
repoBackupStorageName := backupStorageNameFromRepo(backupStorages, repo)
repoBackupStorageName := backupStorageNameFromRepo(backupStorages, repo, db.Namespace)
for eb := p.backupSchedulesToBeReconciled.Front(); eb != nil; eb = ebNext {
// Save the next element because we might remove the current one
ebNext = eb.Next()
Expand All @@ -293,7 +293,7 @@ func (p *pgReposReconciler) reconcileRepos(
// Keep track of backup storages which are already in use by a repo
p.backupStoragesInRepos[backupSchedule.BackupStorageName] = struct{}{}

p.addRepoToPGGlobal(backupStorages[repo.Name].VerifyTLS, repo.Name, backupStorages[repo.Name].ForcePathStyle, &backupSchedule.RetentionCopies, db)
p.addRepoToPGGlobal(backupStorages[repo.Name].Spec.VerifyTLS, repo.Name, backupStorages[repo.Name].Spec.ForcePathStyle, &backupSchedule.RetentionCopies, db)
err := updatePGIni(p.pgBackRestSecretIni, backupStoragesSecrets[backupSchedule.BackupStorageName], repo)
if err != nil {
return errors.Join(err, errors.New("failed to add backup storage credentials to PGBackrest secret data"))
Expand All @@ -306,7 +306,7 @@ func (p *pgReposReconciler) reconcileRepos(
}

func (p *pgReposReconciler) addNewSchedules( //nolint:gocognit
backupStorages map[string]everestv1alpha1.BackupStorageSpec,
backupStorages map[string]everestv1alpha1.BackupStorage,
backupStoragesSecrets map[string]*corev1.Secret,
db *everestv1alpha1.DatabaseCluster,
) error {
Expand All @@ -332,7 +332,7 @@ func (p *pgReposReconciler) addNewSchedules( //nolint:gocognit
if !ok {
return fmt.Errorf("failed to cast repo %v", er.Value)
}
repoBackupStorageName := backupStorageNameFromRepo(backupStorages, repo)
repoBackupStorageName := backupStorageNameFromRepo(backupStorages, repo, db.Namespace)
if backupSchedule.BackupStorageName != repoBackupStorageName ||
repo.BackupSchedules != nil {
continue
Expand All @@ -350,7 +350,7 @@ func (p *pgReposReconciler) addNewSchedules( //nolint:gocognit
if !ok {
return fmt.Errorf("failed to cast repo %v", er.Value)
}
repoBackupStorageName := backupStorageNameFromRepo(backupStorages, repo)
repoBackupStorageName := backupStorageNameFromRepo(backupStorages, repo, db.Namespace)
if backupSchedule.BackupStorageName != repoBackupStorageName ||
repo.BackupSchedules != nil {
continue
Expand All @@ -370,7 +370,7 @@ func (p *pgReposReconciler) addNewSchedules( //nolint:gocognit
}
}

repo, err := genPGBackrestRepo(repoName, backupStorage, &backupSchedule.Schedule)
repo, err := genPGBackrestRepo(repoName, backupStorage.Spec, &backupSchedule.Schedule)
if err != nil {
return err
}
Expand All @@ -379,7 +379,7 @@ func (p *pgReposReconciler) addNewSchedules( //nolint:gocognit
// Keep track of backup storages which are already in use by a repo
p.backupStoragesInRepos[backupSchedule.BackupStorageName] = struct{}{}

p.addRepoToPGGlobal(backupStorage.VerifyTLS, repo.Name, backupStorages[repo.Name].ForcePathStyle, &backupSchedule.RetentionCopies, db)
p.addRepoToPGGlobal(backupStorage.Spec.VerifyTLS, repo.Name, backupStorages[repo.Name].Spec.ForcePathStyle, &backupSchedule.RetentionCopies, db)
err = updatePGIni(p.pgBackRestSecretIni, backupStoragesSecrets[backupSchedule.BackupStorageName], repo)
if err != nil {
return errors.Join(err, errors.New("failed to add backup storage credentials to PGBackrest secret data"))
Expand Down
Loading

0 comments on commit 0133462

Please sign in to comment.