From b8e84091ed41a962fe76195f2638d90739c3b786 Mon Sep 17 00:00:00 2001 From: Pavel Okhlopkov <36456348+ldmonster@users.noreply.github.com> Date: Tue, 28 Jan 2025 14:56:28 +0300 Subject: [PATCH] [deckhouse-controller] Separate deckhouse release reconcile process from generic updater (#11129) Signed-off-by: Pavel Okhlopkov Signed-off-by: Mikhail Scherba Signed-off-by: Mikhail Scherba <41360396+miklezzzz@users.noreply.github.com> Co-authored-by: Pavel Okhlopkov Co-authored-by: Mikhail Scherba Co-authored-by: Mikhail Scherba <41360396+miklezzzz@users.noreply.github.com> --- .../cmd/deckhouse-controller/main.go | 2 + .../apis/deckhouse.io/v1alpha1/annotation.go | 22 - .../v1alpha1/deckhouse_release.go | 68 +- .../deckhouse.io/v1alpha1/module_release.go | 2 + .../pkg/controller/controller.go | 4 +- .../pkg/controller/ctrlutils/options.go | 50 ++ .../pkg/controller/ctrlutils/update.go | 112 +++ .../deckhouse-release/check_release.go | 122 ++- .../deckhouse-release/check_release_test.go | 96 ++- .../cleanup_deckhouse_release.go | 10 +- .../deckhouse-release/client_test.go | 17 +- .../deckhouse-release/controller.go | 713 ++++++++++++++---- .../deckhouse-release/controller_test.go | 131 +++- ...-minor-releases-version-more-than-one.yaml | 66 ++ .../testdata/few-minor-releases.yaml | 75 ++ .../testdata/forced-few-minor-releases.yaml | 77 ++ .../golden/applied-now-postponed-release.yaml | 2 +- .../apply-now-autopatch-mode-is-set.yaml | 2 +- ...ckhouse-previous-release-is-not-ready.yaml | 6 +- ...apply-now-manual-approval-mode-is-set.yaml | 2 +- ...o-deploy-patch-release-in-manual-mode.yaml | 9 +- .../golden/auto-patch-mode-minor-release.yaml | 5 +- .../testdata/golden/auto-patch-mode.yaml | 5 +- .../golden/auto-patch-patch-update.yaml | 4 +- ...ckhouse-previous-release-is-not-ready.yaml | 7 +- .../testdata/golden/disruption-release.yaml | 7 +- .../golden/existed-release-suspended.yaml | 20 + ...-minor-releases-version-more-than-one.yaml | 105 +++ .../testdata/golden/few-minor-releases.yaml | 129 ++++ .../testdata/golden/few-patch-releases.yaml | 26 +- .../golden/forced-few-minor-releases.yaml | 123 +++ .../testdata/golden/forced-release.yaml | 6 +- .../golden/have-canary-release-wave-0.yaml | 10 +- .../golden/have-canary-release-wave-4.yaml | 11 +- .../golden/have-new-deckhouse-image.yaml | 10 +- .../golden/image-hash-not-changed.yaml | 20 + .../golden/inherit-release-cooldown.yaml | 11 +- .../golden/manual-approval-mode-is-set.yaml | 5 +- ...ual-approval-mode-with-canary-process.yaml | 5 +- .../testdata/golden/manual-mode.yaml | 5 +- .../golden/new-release-suspended.yaml | 11 +- .../golden/notification-basic-auth.yaml | 2 +- .../notification-bearer-token-auth.yaml | 2 +- ...ter-time-is-after-notification-period.yaml | 3 +- .../golden/patch-out-of-update-window.yaml | 2 +- .../patch-release-has-own-cooldown.yaml | 11 +- .../golden/patch-release-notification.yaml | 2 +- ...th-apply-now-annotation-out-of-window.yaml | 2 +- ...g-manual-release-on-cluster-bootstrap.yaml | 5 +- .../testdata/golden/postponed-release.yaml | 7 +- .../testdata/golden/release-has-canary.yaml | 11 +- .../testdata/golden/release-has-cooldown.yaml | 11 +- .../golden/release-has-disruptions.yaml | 12 +- .../golden/release-has-requirements.yaml | 13 +- ...-with-apply-now-annotation-disruption.yaml | 2 +- ...th-apply-now-annotation-out-of-window.yaml | 2 +- .../golden/release-with-changelog.yaml | 30 +- .../release-with-notification-settings.yaml | 2 +- ...restore-absent-releases-from-registry.yaml | 6 + .../golden/resume-suspended-release.yaml | 20 + ...a-manual-mode-should-not-change-state.yaml | 5 +- .../step-by-step-update-successfully.yaml | 4 + .../testdata/golden/suspend-release.yaml | 4 +- .../golden/unknown-mode-minor-release.yaml | 5 +- ...hout-configuring-notification-webhook.yaml | 2 +- .../golden/update-out-of-day-window.yaml | 5 +- .../testdata/golden/update-out-of-window.yaml | 5 +- .../golden/update-release-is-deployed.yaml | 2 +- .../updater/deploy_delay_reason.go | 157 ++++ .../updater/deploy_delay_reason_test.go | 61 ++ .../updater/deploy_time_checker.go | 259 +++++++ .../deckhouse-release/updater/metrics.go | 10 +- .../deckhouse-release/updater/release.go | 1 + .../updater/release_notifier.go | 166 ++++ .../updater/requirements_checker.go | 290 +++++++ .../updater/update_task_calculator.go | 257 +++++++ .../deckhouse-release/updater/updater.go | 209 ----- .../module-controllers/release/updater.go | 6 +- .../module-controllers/utils/utils.go | 36 +- deckhouse-controller/pkg/registry/registry.go | 5 +- go_lib/updater/error.go | 17 +- go_lib/updater/metrics.go | 25 +- go_lib/updater/notification.go | 4 + go_lib/updater/settings.go | 12 + go_lib/updater/updater.go | 25 +- 85 files changed, 3159 insertions(+), 671 deletions(-) delete mode 100644 deckhouse-controller/pkg/apis/deckhouse.io/v1alpha1/annotation.go create mode 100644 deckhouse-controller/pkg/controller/ctrlutils/options.go create mode 100644 deckhouse-controller/pkg/controller/ctrlutils/update.go create mode 100644 deckhouse-controller/pkg/controller/deckhouse-release/testdata/few-minor-releases-version-more-than-one.yaml create mode 100644 deckhouse-controller/pkg/controller/deckhouse-release/testdata/few-minor-releases.yaml create mode 100644 deckhouse-controller/pkg/controller/deckhouse-release/testdata/forced-few-minor-releases.yaml create mode 100644 deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/few-minor-releases-version-more-than-one.yaml create mode 100644 deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/few-minor-releases.yaml create mode 100644 deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/forced-few-minor-releases.yaml create mode 100644 deckhouse-controller/pkg/controller/deckhouse-release/updater/deploy_delay_reason.go create mode 100644 deckhouse-controller/pkg/controller/deckhouse-release/updater/deploy_delay_reason_test.go create mode 100644 deckhouse-controller/pkg/controller/deckhouse-release/updater/deploy_time_checker.go create mode 100644 deckhouse-controller/pkg/controller/deckhouse-release/updater/release_notifier.go create mode 100644 deckhouse-controller/pkg/controller/deckhouse-release/updater/requirements_checker.go create mode 100644 deckhouse-controller/pkg/controller/deckhouse-release/updater/update_task_calculator.go delete mode 100644 deckhouse-controller/pkg/controller/deckhouse-release/updater/updater.go diff --git a/deckhouse-controller/cmd/deckhouse-controller/main.go b/deckhouse-controller/cmd/deckhouse-controller/main.go index 7ec8ee7cb0..1aaccdd01a 100644 --- a/deckhouse-controller/cmd/deckhouse-controller/main.go +++ b/deckhouse-controller/cmd/deckhouse-controller/main.go @@ -28,6 +28,7 @@ import ( sh_debug "github.com/flant/shell-operator/pkg/debug" "gopkg.in/alecthomas/kingpin.v2" + "github.com/deckhouse/deckhouse/deckhouse-controller/pkg/controller" "github.com/deckhouse/deckhouse/deckhouse-controller/pkg/debug" "github.com/deckhouse/deckhouse/deckhouse-controller/pkg/helpers" "github.com/deckhouse/deckhouse/deckhouse-controller/pkg/registry" @@ -58,6 +59,7 @@ const ( func main() { sh_app.Version = ShellOperatorVersion ad_app.Version = AddonOperatorVersion + controller.DeckhouseVersion = DeckhouseVersion FileName := filepath.Base(os.Args[0]) diff --git a/deckhouse-controller/pkg/apis/deckhouse.io/v1alpha1/annotation.go b/deckhouse-controller/pkg/apis/deckhouse.io/v1alpha1/annotation.go deleted file mode 100644 index eea34c5d07..0000000000 --- a/deckhouse-controller/pkg/apis/deckhouse.io/v1alpha1/annotation.go +++ /dev/null @@ -1,22 +0,0 @@ -/* -Copyright 2024 Flant JSC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1alpha1 - -const ( - ModuleReleaseApprovalAnnotation = "modules.deckhouse.io/approved" - DeckhouseReleaseApprovalAnnotation = "release.deckhouse.io/approved" -) diff --git a/deckhouse-controller/pkg/apis/deckhouse.io/v1alpha1/deckhouse_release.go b/deckhouse-controller/pkg/apis/deckhouse.io/v1alpha1/deckhouse_release.go index e781ac33ab..8cbbf2a419 100644 --- a/deckhouse-controller/pkg/apis/deckhouse.io/v1alpha1/deckhouse_release.go +++ b/deckhouse-controller/pkg/apis/deckhouse.io/v1alpha1/deckhouse_release.go @@ -24,6 +24,30 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" ) +const ( + DeckhouseReleasePhasePending = "Pending" + DeckhouseReleasePhaseDeployed = "Deployed" + DeckhouseReleasePhaseSuperseded = "Superseded" + DeckhouseReleasePhaseSuspended = "Suspended" + DeckhouseReleasePhaseSkipped = "Skipped" + + DeckhouseReleaseApprovalAnnotation = "release.deckhouse.io/approved" + DeckhouseReleaseAnnotationIsUpdating = "release.deckhouse.io/isUpdating" + DeckhouseReleaseAnnotationNotified = "release.deckhouse.io/notified" + DeckhouseReleaseAnnotationApplyNow = "release.deckhouse.io/apply-now" + DeckhouseReleaseAnnotationApplyAfter = "release.deckhouse.io/applyAfter" + DeckhouseReleaseAnnotationDisruptionApproved = "release.deckhouse.io/disruption-approved" + DeckhouseReleaseAnnotationForce = "release.deckhouse.io/force" + DeckhouseReleaseAnnotationSuspended = "release.deckhouse.io/suspended" + DeckhouseReleaseAnnotationNotificationTimeShift = "release.deckhouse.io/notification-time-shift" + DeckhouseReleaseAnnotationDryrun = "dryrun" + DeckhouseReleaseAnnotationTriggeredByDryrun = "triggered_by_dryrun" + DeckhouseReleaseAnnotationCurrentRestored = "release.deckhouse.io/current-restored" + + // TODO: remove in entire code + DeckhouseReleaseAnnotationCooldown = "release.deckhouse.io/cooldown" +) + var DeckhouseReleaseGVK = schema.GroupVersionKind{ Group: SchemeGroupVersion.Group, Version: SchemeGroupVersion.Version, @@ -69,9 +93,10 @@ func (in *DeckhouseRelease) GetChangelogLink() string { return in.Spec.ChangelogLink } +// TODO: remove cooldown from entire code func (in *DeckhouseRelease) GetCooldownUntil() *time.Time { cooldown := new(time.Time) - if v, ok := in.Annotations["release.deckhouse.io/cooldown"]; ok { + if v, ok := in.Annotations[DeckhouseReleaseAnnotationCooldown]; ok { cd, err := time.Parse(time.RFC3339, v) if err == nil { cooldown = &cd @@ -86,7 +111,7 @@ func (in *DeckhouseRelease) GetDisruptions() []string { } func (in *DeckhouseRelease) GetDisruptionApproved() bool { - v, ok := in.Annotations["release.deckhouse.io/disruption-approved"] + v, ok := in.Annotations[DeckhouseReleaseAnnotationDisruptionApproved] return ok && v == "true" } @@ -95,12 +120,22 @@ func (in *DeckhouseRelease) GetPhase() string { } func (in *DeckhouseRelease) GetForce() bool { - v, ok := in.Annotations["release.deckhouse.io/force"] + v, ok := in.Annotations[DeckhouseReleaseAnnotationForce] return ok && v == "true" } func (in *DeckhouseRelease) GetApplyNow() bool { - v, ok := in.Annotations["release.deckhouse.io/apply-now"] + v, ok := in.Annotations[DeckhouseReleaseAnnotationApplyNow] + return ok && v == "true" +} + +func (in *DeckhouseRelease) GetIsUpdating() bool { + v, ok := in.Annotations[DeckhouseReleaseAnnotationIsUpdating] + return ok && v == "true" +} + +func (in *DeckhouseRelease) GetNotified() bool { + v, ok := in.Annotations[DeckhouseReleaseAnnotationNotified] return ok && v == "true" } @@ -113,7 +148,7 @@ func (in *DeckhouseRelease) SetApprovedStatus(val bool) { } func (in *DeckhouseRelease) GetSuspend() bool { - v, ok := in.Annotations["release.deckhouse.io/suspended"] + v, ok := in.Annotations[DeckhouseReleaseAnnotationSuspended] return ok && v == "true" } @@ -123,11 +158,7 @@ func (in *DeckhouseRelease) GetManuallyApproved() bool { } v, ok := in.Annotations[DeckhouseReleaseApprovalAnnotation] - if ok { - return v == "true" - } - - return in.Approved + return ok && v == "true" } func (in *DeckhouseRelease) GetMessage() string { @@ -135,7 +166,22 @@ func (in *DeckhouseRelease) GetMessage() string { } func (in *DeckhouseRelease) GetNotificationShift() bool { - v, ok := in.Annotations["release.deckhouse.io/notification-time-shift"] + v, ok := in.Annotations[DeckhouseReleaseAnnotationNotificationTimeShift] + return ok && v == "true" +} + +func (in *DeckhouseRelease) GetDryRun() bool { + v, ok := in.Annotations[DeckhouseReleaseAnnotationDryrun] + return ok && v == "true" +} + +func (in *DeckhouseRelease) GetTriggeredByDryRun() bool { + v, ok := in.Annotations[DeckhouseReleaseAnnotationTriggeredByDryrun] + return ok && v == "true" +} + +func (in *DeckhouseRelease) GetCurrentRestored() bool { + v, ok := in.Annotations[DeckhouseReleaseAnnotationCurrentRestored] return ok && v == "true" } diff --git a/deckhouse-controller/pkg/apis/deckhouse.io/v1alpha1/module_release.go b/deckhouse-controller/pkg/apis/deckhouse.io/v1alpha1/module_release.go index 9a8a7f06c9..9fb5fabd22 100644 --- a/deckhouse-controller/pkg/apis/deckhouse.io/v1alpha1/module_release.go +++ b/deckhouse-controller/pkg/apis/deckhouse.io/v1alpha1/module_release.go @@ -36,6 +36,8 @@ const ( ModuleReleasePhaseSuspended = "Suspended" ModuleReleasePhaseSkipped = "Skipped" + ModuleReleaseApprovalAnnotation = "modules.deckhouse.io/approved" + ModuleReleaseAnnotationApplyNow = "release.deckhouse.io/apply-now" ModuleReleaseAnnotationRegistrySpecChanged = "modules.deckhouse.io/registry-spec-changed" diff --git a/deckhouse-controller/pkg/controller/controller.go b/deckhouse-controller/pkg/controller/controller.go index 11466f9aaa..13925fe113 100644 --- a/deckhouse-controller/pkg/controller/controller.go +++ b/deckhouse-controller/pkg/controller/controller.go @@ -62,6 +62,8 @@ import ( "github.com/deckhouse/deckhouse/pkg/log" ) +var DeckhouseVersion string + const ( docsLeaseLabel = "deckhouse.io/documentation-builder-sync" @@ -225,7 +227,7 @@ func NewDeckhouseController(ctx context.Context, version string, operator *addon loader := moduleloader.New(runtimeManager.GetClient(), version, operator.ModuleManager.ModulesDir, operator.ModuleManager.GlobalHooksDir, dc, embeddedPolicy, logger.Named("module-loader")) operator.ModuleManager.SetModuleLoader(loader) - err = deckhouserelease.NewDeckhouseReleaseController(ctx, runtimeManager, dc, operator.ModuleManager, settingsContainer, operator.MetricStorage, preflightCountDown, logger.Named("deckhouse-release-controller")) + err = deckhouserelease.NewDeckhouseReleaseController(ctx, runtimeManager, dc, operator.ModuleManager, settingsContainer, operator.MetricStorage, preflightCountDown, DeckhouseVersion, logger.Named("deckhouse-release-controller")) if err != nil { return nil, fmt.Errorf("create deckhouse release controller: %w", err) } diff --git a/deckhouse-controller/pkg/controller/ctrlutils/options.go b/deckhouse-controller/pkg/controller/ctrlutils/options.go new file mode 100644 index 0000000000..76677d6e1f --- /dev/null +++ b/deckhouse-controller/pkg/controller/ctrlutils/options.go @@ -0,0 +1,50 @@ +// Copyright 2024 Flant JSC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ctrlutils + +import ( + "k8s.io/apimachinery/pkg/util/wait" +) + +type UpdateOption func(optionsApplier UpdateOptionApplier) + +func (opt UpdateOption) Apply(o UpdateOptionApplier) { + opt(o) +} + +type UpdateOptionApplier interface { + WithOnErrorBackoff(b *wait.Backoff) + WithRetryOnConflictBackoff(b *wait.Backoff) + + withStatusUpdate() +} + +func WithOnErrorBackoff(b *wait.Backoff) UpdateOption { + return func(optionsApplier UpdateOptionApplier) { + optionsApplier.WithOnErrorBackoff(b) + } +} + +func WithRetryOnConflictBackoff(b *wait.Backoff) UpdateOption { + return func(optionsApplier UpdateOptionApplier) { + optionsApplier.WithRetryOnConflictBackoff(b) + } +} + +func withStatusUpdate() UpdateOption { + return func(optionsApplier UpdateOptionApplier) { + optionsApplier.withStatusUpdate() + } +} diff --git a/deckhouse-controller/pkg/controller/ctrlutils/update.go b/deckhouse-controller/pkg/controller/ctrlutils/update.go new file mode 100644 index 0000000000..63e93c98e8 --- /dev/null +++ b/deckhouse-controller/pkg/controller/ctrlutils/update.go @@ -0,0 +1,112 @@ +// Copyright 2024 Flant JSC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ctrlutils + +import ( + "context" + "errors" + + "k8s.io/apimachinery/pkg/api/equality" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/util/retry" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type MutateFn func() error + +type UpdateOptions struct { + OnErrorBackoff wait.Backoff + RetryOnConflictBackoff wait.Backoff + + statusUpdate bool +} + +func (o *UpdateOptions) WithOnErrorBackoff(b *wait.Backoff) { + if b == nil { + return + } + + o.OnErrorBackoff = *b +} + +func (o *UpdateOptions) WithRetryOnConflictBackoff(b *wait.Backoff) { + if b == nil { + return + } + + o.RetryOnConflictBackoff = *b +} + +func (o *UpdateOptions) withStatusUpdate() { + o.statusUpdate = true +} + +var ErrCanNotMutateNameOrNamespace = errors.New("MutateFn cannot mutate object name and/or object namespace") + +func UpdateWithRetry(ctx context.Context, c client.Client, obj client.Object, f MutateFn, opts ...UpdateOption) error { + options := &UpdateOptions{ + OnErrorBackoff: retry.DefaultRetry, + RetryOnConflictBackoff: retry.DefaultRetry, + } + + for _, fn := range opts { + fn.Apply(options) + } + + key := client.ObjectKeyFromObject(obj) + + return retry.OnError(options.OnErrorBackoff, apierrors.IsServiceUnavailable, func() error { + return retry.RetryOnConflict(options.RetryOnConflictBackoff, func() error { + if err := c.Get(ctx, key, obj); err != nil { + return client.IgnoreNotFound(err) + } + + existing := obj.DeepCopyObject() + if err := mutate(f, key, obj); err != nil { + return err + } + + if equality.Semantic.DeepEqual(existing, obj) { + return nil + } + + if options.statusUpdate { + return c.Status().Update(ctx, obj) + } + + return c.Update(ctx, obj) + }) + }) +} + +func UpdateStatusWithRetry(ctx context.Context, c client.Client, obj client.Object, f MutateFn, opts ...UpdateOption) error { + opts = append(opts, withStatusUpdate()) + + return UpdateWithRetry(ctx, c, obj, f, opts...) +} + +// mutate wraps a MutateFn and applies validation to its result. +func mutate(f MutateFn, key client.ObjectKey, obj client.Object) error { + if err := f(); err != nil { + return err + } + + if newKey := client.ObjectKeyFromObject(obj); key != newKey { + return ErrCanNotMutateNameOrNamespace + } + + return nil +} diff --git a/deckhouse-controller/pkg/controller/deckhouse-release/check_release.go b/deckhouse-controller/pkg/controller/deckhouse-release/check_release.go index bacb2fcc70..0b51ace82e 100644 --- a/deckhouse-controller/pkg/controller/deckhouse-release/check_release.go +++ b/deckhouse-controller/pkg/controller/deckhouse-release/check_release.go @@ -23,6 +23,7 @@ import ( "encoding/json" "fmt" "io" + "log/slog" "path" "regexp" "sort" @@ -55,9 +56,6 @@ const ( func (r *deckhouseReleaseReconciler) checkDeckhouseReleaseLoop(ctx context.Context) { wait.UntilWithContext(ctx, func(ctx context.Context) { - if r.updateSettings.Get().ReleaseChannel == "" { - return - } err := r.checkDeckhouseRelease(ctx) if err != nil { r.logger.Errorf("check Deckhouse release: %s", err) @@ -67,7 +65,7 @@ func (r *deckhouseReleaseReconciler) checkDeckhouseReleaseLoop(ctx context.Conte func (r *deckhouseReleaseReconciler) checkDeckhouseRelease(ctx context.Context) error { if r.updateSettings.Get().ReleaseChannel == "" { - r.logger.Debug("Release channel does not set.") + r.logger.Debug("Release channel isn't set.") return nil } @@ -90,17 +88,16 @@ func (r *deckhouseReleaseReconciler) checkDeckhouseRelease(ctx context.Context) imagesRegistry string ) if registrySecret != nil { - drs, _ := utils.ParseDeckhouseRegistrySecret(registrySecret.Data) rconf := &utils.RegistryConfig{ - DockerConfig: drs.DockerConfig, - Scheme: drs.Scheme, - CA: drs.CA, + DockerConfig: registrySecret.DockerConfig, + Scheme: registrySecret.Scheme, + CA: registrySecret.CA, UserAgent: r.clusterUUID, } opts = utils.GenerateRegistryOptions(rconf, r.logger) - imagesRegistry = drs.ImageRegistry + imagesRegistry = registrySecret.ImageRegistry } releaseChecker, err := NewDeckhouseReleaseChecker(opts, r.logger, r.dc, r.moduleManager, imagesRegistry, releaseChannelName) @@ -113,6 +110,34 @@ func (r *deckhouseReleaseReconciler) checkDeckhouseRelease(ctx context.Context) return err } + var ( + releases v1alpha1.DeckhouseReleaseList + currentDeployedRelease *v1alpha1.DeckhouseRelease + ) + err = r.client.List(ctx, &releases) + if err != nil { + return fmt.Errorf("get deckhouse releases: %w", err) + } + + pointerReleases := make([]*v1alpha1.DeckhouseRelease, 0, len(releases.Items)) + for _, r := range releases.Items { + if r.GetPhase() == v1alpha1.DeckhouseReleasePhaseDeployed { + // no deployed release was found or there is more than one deployed release (get the latest) + if currentDeployedRelease == nil || r.GetVersion().GreaterThan(currentDeployedRelease.GetVersion()) { + currentDeployedRelease = &r + } + } + pointerReleases = append(pointerReleases, &r) + } + sort.Sort(sort.Reverse(updater.ByVersion[*v1alpha1.DeckhouseRelease](pointerReleases))) + + // restore current deployed release if no deployed releases found + if currentDeployedRelease == nil { + if err := r.restoreCurrentDeployedRelease(ctx, releaseChecker, r.deckhouseVersion); err != nil { + return fmt.Errorf("restore current deployed release: %w", err) + } + } + // no new image found if newImageHash == "" { return nil @@ -132,18 +157,6 @@ func (r *deckhouseReleaseReconciler) checkDeckhouseRelease(ctx context.Context) return fmt.Errorf("parse semver: %w", err) } r.releaseVersionImageHash = newImageHash - - var releases v1alpha1.DeckhouseReleaseList - err = r.client.List(ctx, &releases) - if err != nil { - return fmt.Errorf("get deckhouse releases: %w", err) - } - - pointerReleases := make([]*v1alpha1.DeckhouseRelease, 0, len(releases.Items)) - for _, r := range releases.Items { - pointerReleases = append(pointerReleases, &r) - } - sort.Sort(sort.Reverse(updater.ByVersion[*v1alpha1.DeckhouseRelease](pointerReleases))) r.metricStorage.Grouped().ExpireGroupMetrics(metricUpdatingFailedGroup) for _, release := range pointerReleases { @@ -151,7 +164,7 @@ func (r *deckhouseReleaseReconciler) checkDeckhouseRelease(ctx context.Context) // GT case release.GetVersion().GreaterThan(newSemver): // cleanup versions which are older than current version in a specified channel and are in a Pending state - if release.Status.Phase == v1alpha1.ModuleReleasePhasePending { + if release.Status.Phase == v1alpha1.DeckhouseReleasePhasePending { err = r.client.Delete(ctx, release, client.PropagationPolicy(metav1.DeletePropagationBackground)) if err != nil { return fmt.Errorf("delete old release: %w", err) @@ -162,7 +175,7 @@ func (r *deckhouseReleaseReconciler) checkDeckhouseRelease(ctx context.Context) case release.GetVersion().Equal(newSemver): r.logger.Debugf("Release with version %s already exists", release.GetVersion()) switch release.Status.Phase { - case v1alpha1.ModuleReleasePhasePending, "": + case v1alpha1.DeckhouseReleasePhasePending, "": if releaseChecker.releaseMetadata.Suspend { patch := client.RawPatch(types.MergePatchType, buildSuspendAnnotation(releaseChecker.releaseMetadata.Suspend)) err := r.client.Patch(ctx, release, patch) @@ -176,7 +189,7 @@ func (r *deckhouseReleaseReconciler) checkDeckhouseRelease(ctx context.Context) } } - case v1alpha1.ModuleReleasePhaseSuspended: + case v1alpha1.DeckhouseReleasePhaseSuspended: if !releaseChecker.releaseMetadata.Suspend { patch := client.RawPatch(types.MergePatchType, buildSuspendAnnotation(releaseChecker.releaseMetadata.Suspend)) err := r.client.Patch(ctx, release, patch) @@ -235,14 +248,48 @@ func (r *deckhouseReleaseReconciler) checkDeckhouseRelease(ctx context.Context) } } - // if there are no releases in the cluster, we apply the latest release - if len(pointerReleases) == 0 { - err = r.createRelease(ctx, releaseChecker, cooldownUntil, notificationShiftTime) - if err != nil { - return fmt.Errorf("create release %s: %w", releaseChecker.releaseMetadata.Version, err) + return nil +} + +func (r *deckhouseReleaseReconciler) restoreCurrentDeployedRelease(ctx context.Context, releaseChecker *DeckhouseReleaseChecker, tag string) error { + var releaseMetadata *ReleaseMetadata + + if image, err := releaseChecker.registryClient.Image(ctx, tag); err != nil { + r.logger.Warn("couldn't get current deployed release's image from registry", slog.String("image", tag), log.Err(err)) + } else { + if releaseMetadata, err = releaseChecker.fetchReleaseMetadata(image); err != nil { + r.logger.Warn("couldn't fetch current deployed release's image metadata", slog.String("image", tag), log.Err(err)) } } - return nil + + release := &v1alpha1.DeckhouseRelease{ + TypeMeta: metav1.TypeMeta{ + Kind: "DeckhouseRelease", + APIVersion: "deckhouse.io/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: tag, + Annotations: map[string]string{ + v1alpha1.DeckhouseReleaseAnnotationIsUpdating: "false", + v1alpha1.DeckhouseReleaseAnnotationNotified: "false", + v1alpha1.DeckhouseReleaseAnnotationCurrentRestored: "true", + }, + Labels: map[string]string{ + "heritage": "deckhouse", + }, + }, + Spec: v1alpha1.DeckhouseReleaseSpec{ + Version: tag, + }, + Approved: true, + } + + if releaseMetadata != nil { + release.Spec.Requirements = releaseMetadata.Requirements + release.Spec.ChangelogLink = fmt.Sprintf("https://github.com/deckhouse/deckhouse/releases/tag/%s", releaseMetadata.Version) + } + + return client.IgnoreAlreadyExists(r.client.Create(ctx, release)) } func (r *deckhouseReleaseReconciler) createRelease(ctx context.Context, releaseChecker *DeckhouseReleaseChecker, @@ -284,8 +331,11 @@ func (r *deckhouseReleaseReconciler) createRelease(ctx context.Context, releaseC ObjectMeta: metav1.ObjectMeta{ Name: releaseChecker.releaseMetadata.Version, Annotations: map[string]string{ - "release.deckhouse.io/isUpdating": "false", - "release.deckhouse.io/notified": "false", + v1alpha1.DeckhouseReleaseAnnotationIsUpdating: "false", + v1alpha1.DeckhouseReleaseAnnotationNotified: "false", + }, + Labels: map[string]string{ + "heritage": "deckhouse", }, }, Spec: v1alpha1.DeckhouseReleaseSpec{ @@ -300,13 +350,13 @@ func (r *deckhouseReleaseReconciler) createRelease(ctx context.Context, releaseC } if releaseChecker.releaseMetadata.Suspend { - release.ObjectMeta.Annotations["release.deckhouse.io/suspended"] = "true" + release.ObjectMeta.Annotations[v1alpha1.DeckhouseReleaseAnnotationSuspended] = "true" } if cooldownUntil != nil { - release.ObjectMeta.Annotations["release.deckhouse.io/cooldown"] = cooldownUntil.UTC().Format(time.RFC3339) + release.ObjectMeta.Annotations[v1alpha1.DeckhouseReleaseAnnotationCooldown] = cooldownUntil.UTC().Format(time.RFC3339) } if notificationShiftTime != nil { - release.ObjectMeta.Annotations["release.deckhouse.io/notification-time-shift"] = "true" + release.ObjectMeta.Annotations[v1alpha1.DeckhouseReleaseAnnotationNotificationTimeShift] = "true" } return client.IgnoreAlreadyExists(r.client.Create(ctx, release)) @@ -649,7 +699,7 @@ func buildSuspendAnnotation(suspend bool) []byte { p := map[string]interface{}{ "metadata": map[string]interface{}{ "annotations": map[string]interface{}{ - "release.deckhouse.io/suspended": annotationValue, + v1alpha1.DeckhouseReleaseAnnotationSuspended: annotationValue, }, }, } diff --git a/deckhouse-controller/pkg/controller/deckhouse-release/check_release_test.go b/deckhouse-controller/pkg/controller/deckhouse-release/check_release_test.go index 592523b9c9..abf6eb487c 100644 --- a/deckhouse-controller/pkg/controller/deckhouse-release/check_release_test.go +++ b/deckhouse-controller/pkg/controller/deckhouse-release/check_release_test.go @@ -70,19 +70,28 @@ func (suite *ControllerTestSuite) TestCheckDeckhouseRelease() { } }` + var testDeckhouseVersionImage = &fake.FakeImage{ + ManifestStub: ManifestStub, + LayersStub: func() ([]v1.Layer, error) { + return []v1.Layer{&fakeLayer{}, &fakeLayer{ + FilesContent: map[string]string{ + "version.json": fmt.Sprintf("{`version`: `%s`}", testDeckhouseVersion), + }}}, nil + }} + suite.Run("Have new deckhouse image", func() { - dependency.TestDC.CRClient.ImageMock.Return( - &fake.FakeImage{ - ManifestStub: ManifestStub, - LayersStub: func() ([]v1.Layer, error) { - return []v1.Layer{&fakeLayer{}, &fakeLayer{ - FilesContent: map[string]string{`version.json`: `{"version": "v1.25.3"}`}}, - }, nil - }, - DigestStub: func() (v1.Hash, error) { - return v1.NewHash("sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b777") - }, - }, nil, + dependency.TestDC.CRClient.ImageMock.When(testDeckhouseVersion).Then(testDeckhouseVersionImage, nil) + dependency.TestDC.CRClient.ImageMock.When("stable").Then(&fake.FakeImage{ + ManifestStub: ManifestStub, + LayersStub: func() ([]v1.Layer, error) { + return []v1.Layer{&fakeLayer{}, &fakeLayer{ + FilesContent: map[string]string{`version.json`: `{"version": "v1.25.3"}`}}, + }, nil + }, + DigestStub: func() (v1.Hash, error) { + return v1.NewHash("sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b777") + }, + }, nil, ) suite.setupController("have-new-deckhouse-image.yaml", initValues, embeddedMUP) @@ -91,20 +100,19 @@ func (suite *ControllerTestSuite) TestCheckDeckhouseRelease() { }) suite.Run("Have canary release wave 0", func() { - dependency.TestDC.CRClient.ImageMock.Return( - &fake.FakeImage{ - ManifestStub: ManifestStub, - LayersStub: func() ([]v1.Layer, error) { - return []v1.Layer{&fakeLayer{}, &fakeLayer{ - FilesContent: map[string]string{ - `version.json`: `{"version": "v1.25.0", "canary": {"stable": {"enabled": true, "waves": 5, "interval": "6m"}}}`, - }}}, nil - }, - DigestStub: func() (v1.Hash, error) { - return v1.NewHash("sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855") - }, - }, nil, - ) + dependency.TestDC.CRClient.ImageMock.When(testDeckhouseVersion).Then(testDeckhouseVersionImage, nil) + dependency.TestDC.CRClient.ImageMock.When("stable").Then(&fake.FakeImage{ + ManifestStub: ManifestStub, + LayersStub: func() ([]v1.Layer, error) { + return []v1.Layer{&fakeLayer{}, &fakeLayer{ + FilesContent: map[string]string{ + `version.json`: `{"version": "v1.25.0", "canary": {"stable": {"enabled": true, "waves": 5, "interval": "6m"}}}`, + }}}, nil + }, + DigestStub: func() (v1.Hash, error) { + return v1.NewHash("sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855") + }, + }, nil) suite.setupController("have-canary-release-wave-0.yaml", initValues, embeddedMUP) err := suite.ctr.checkDeckhouseRelease(ctx) @@ -112,7 +120,8 @@ func (suite *ControllerTestSuite) TestCheckDeckhouseRelease() { }) suite.Run("Have canary release wave 4", func() { - dependency.TestDC.CRClient.ImageMock.Return(&fake.FakeImage{ + dependency.TestDC.CRClient.ImageMock.When(testDeckhouseVersion).Then(testDeckhouseVersionImage, nil) + dependency.TestDC.CRClient.ImageMock.When("stable").Then(&fake.FakeImage{ ManifestStub: ManifestStub, LayersStub: func() ([]v1.Layer, error) { return []v1.Layer{&fakeLayer{}, &fakeLayer{ @@ -130,7 +139,8 @@ func (suite *ControllerTestSuite) TestCheckDeckhouseRelease() { }) suite.Run("Existed release suspended", func() { - dependency.TestDC.CRClient.ImageMock.Return(&fake.FakeImage{ + dependency.TestDC.CRClient.ImageMock.When(testDeckhouseVersion).Then(testDeckhouseVersionImage, nil) + dependency.TestDC.CRClient.ImageMock.When("stable").Then(&fake.FakeImage{ ManifestStub: ManifestStub, LayersStub: func() ([]v1.Layer, error) { return []v1.Layer{&fakeLayer{}, &fakeLayer{ @@ -163,7 +173,8 @@ func (suite *ControllerTestSuite) TestCheckDeckhouseRelease() { }) suite.Run("New release suspended", func() { - dependency.TestDC.CRClient.ImageMock.Return(&fake.FakeImage{ + dependency.TestDC.CRClient.ImageMock.When(testDeckhouseVersion).Then(testDeckhouseVersionImage, nil) + dependency.TestDC.CRClient.ImageMock.When("stable").Then(&fake.FakeImage{ ManifestStub: ManifestStub, LayersStub: func() ([]v1.Layer, error) { return []v1.Layer{&fakeLayer{}, &fakeLayer{ @@ -180,7 +191,8 @@ func (suite *ControllerTestSuite) TestCheckDeckhouseRelease() { }) suite.Run("Resume suspended release", func() { - dependency.TestDC.CRClient.ImageMock.Return(&fake.FakeImage{ + dependency.TestDC.CRClient.ImageMock.When(testDeckhouseVersion).Then(testDeckhouseVersionImage, nil) + dependency.TestDC.CRClient.ImageMock.When("stable").Then(&fake.FakeImage{ ManifestStub: ManifestStub, LayersStub: func() ([]v1.Layer, error) { return []v1.Layer{&fakeLayer{}, &fakeLayer{ @@ -197,7 +209,8 @@ func (suite *ControllerTestSuite) TestCheckDeckhouseRelease() { }) suite.Run("Image hash not changed", func() { - dependency.TestDC.CRClient.ImageMock.Return(&fake.FakeImage{ + dependency.TestDC.CRClient.ImageMock.When(testDeckhouseVersion).Then(testDeckhouseVersionImage, nil) + dependency.TestDC.CRClient.ImageMock.When("stable").Then(&fake.FakeImage{ ManifestStub: ManifestStub, LayersStub: func() ([]v1.Layer, error) { return []v1.Layer{&fakeLayer{}, &fakeLayer{ @@ -215,7 +228,8 @@ func (suite *ControllerTestSuite) TestCheckDeckhouseRelease() { }) suite.Run("Release has requirements", func() { - dependency.TestDC.CRClient.ImageMock.Return(&fake.FakeImage{ + dependency.TestDC.CRClient.ImageMock.When(testDeckhouseVersion).Then(testDeckhouseVersionImage, nil) + dependency.TestDC.CRClient.ImageMock.When("stable").Then(&fake.FakeImage{ ManifestStub: ManifestStub, LayersStub: func() ([]v1.Layer, error) { return []v1.Layer{&fakeLayer{}, &fakeLayer{ @@ -234,7 +248,8 @@ func (suite *ControllerTestSuite) TestCheckDeckhouseRelease() { }) suite.Run("Release has canary", func() { - dependency.TestDC.CRClient.ImageMock.Return(&fake.FakeImage{ + dependency.TestDC.CRClient.ImageMock.When(testDeckhouseVersion).Then(testDeckhouseVersionImage, nil) + dependency.TestDC.CRClient.ImageMock.When("stable").Then(&fake.FakeImage{ ManifestStub: ManifestStub, LayersStub: func() ([]v1.Layer, error) { return []v1.Layer{&fakeLayer{}, &fakeLayer{ @@ -262,7 +277,8 @@ func (suite *ControllerTestSuite) TestCheckDeckhouseRelease() { "v1.33.0", "v1.33.1", }, nil) - dependency.TestDC.CRClient.ImageMock.Return(&fake.FakeImage{ + dependency.TestDC.CRClient.ImageMock.When(testDeckhouseVersion).Then(testDeckhouseVersionImage, nil) + dependency.TestDC.CRClient.ImageMock.When("stable").Then(&fake.FakeImage{ ManifestStub: ManifestStub, LayersStub: func() ([]v1.Layer, error) { return []v1.Layer{&fakeLayer{}, &fakeLayer{FilesContent: map[string]string{ @@ -287,7 +303,8 @@ func (suite *ControllerTestSuite) TestCheckDeckhouseRelease() { }) suite.Run("Inherit release cooldown", func() { - dependency.TestDC.CRClient.ImageMock.Return(&fake.FakeImage{ + dependency.TestDC.CRClient.ImageMock.When(testDeckhouseVersion).Then(testDeckhouseVersionImage, nil) + dependency.TestDC.CRClient.ImageMock.When("stable").Then(&fake.FakeImage{ ManifestStub: ManifestStub, LayersStub: func() ([]v1.Layer, error) { return []v1.Layer{&fakeLayer{}, &fakeLayer{FilesContent: map[string]string{ @@ -312,7 +329,8 @@ func (suite *ControllerTestSuite) TestCheckDeckhouseRelease() { }) suite.Run("Patch release has own cooldown", func() { - dependency.TestDC.CRClient.ImageMock.Return(&fake.FakeImage{ + dependency.TestDC.CRClient.ImageMock.When(testDeckhouseVersion).Then(testDeckhouseVersionImage, nil) + dependency.TestDC.CRClient.ImageMock.When("stable").Then(&fake.FakeImage{ ManifestStub: ManifestStub, LayersStub: func() ([]v1.Layer, error) { return []v1.Layer{&fakeLayer{}, &fakeLayer{FilesContent: map[string]string{ @@ -337,7 +355,8 @@ func (suite *ControllerTestSuite) TestCheckDeckhouseRelease() { }) suite.Run("Release has disruptions", func() { - dependency.TestDC.CRClient.ImageMock.Return(&fake.FakeImage{ + dependency.TestDC.CRClient.ImageMock.When(testDeckhouseVersion).Then(testDeckhouseVersionImage, nil) + dependency.TestDC.CRClient.ImageMock.When("stable").Then(&fake.FakeImage{ ManifestStub: ManifestStub, LayersStub: func() ([]v1.Layer, error) { return []v1.Layer{&fakeLayer{}, &fakeLayer{FilesContent: map[string]string{ @@ -380,7 +399,8 @@ global: changelog := fmt.Sprintf(changelogTemplate, "`control-plane`") // global.features[0].description - dependency.TestDC.CRClient.ImageMock.Return(&fake.FakeImage{ + dependency.TestDC.CRClient.ImageMock.When(testDeckhouseVersion).Then(testDeckhouseVersionImage, nil) + dependency.TestDC.CRClient.ImageMock.When("stable").Then(&fake.FakeImage{ ManifestStub: ManifestStub, LayersStub: func() ([]v1.Layer, error) { return []v1.Layer{ diff --git a/deckhouse-controller/pkg/controller/deckhouse-release/cleanup_deckhouse_release.go b/deckhouse-controller/pkg/controller/deckhouse-release/cleanup_deckhouse_release.go index 653a8ca62d..347e409ff3 100644 --- a/deckhouse-controller/pkg/controller/deckhouse-release/cleanup_deckhouse_release.go +++ b/deckhouse-controller/pkg/controller/deckhouse-release/cleanup_deckhouse_release.go @@ -65,13 +65,13 @@ func (r *deckhouseReleaseReconciler) cleanupDeckhouseRelease(ctx context.Context for i, release := range pointerReleases { switch release.Status.Phase { - case v1alpha1.ModuleReleasePhasePending: + case v1alpha1.DeckhouseReleasePhasePending: pendingReleasesIndexes = append(pendingReleasesIndexes, i) - case v1alpha1.ModuleReleasePhaseDeployed: + case v1alpha1.DeckhouseReleasePhaseDeployed: deployedReleasesIndexes = append(deployedReleasesIndexes, i) - case v1alpha1.ModuleReleasePhaseSuperseded, v1alpha1.ModuleReleasePhaseSkipped, v1alpha1.ModuleReleasePhaseSuspended: + case v1alpha1.DeckhouseReleasePhaseSuperseded, v1alpha1.DeckhouseReleasePhaseSkipped, v1alpha1.DeckhouseReleasePhaseSuspended: outdatedReleasesIndexes = append(outdatedReleasesIndexes, i) } } @@ -79,7 +79,7 @@ func (r *deckhouseReleaseReconciler) cleanupDeckhouseRelease(ctx context.Context if len(deployedReleasesIndexes) > 1 { // cleanup releases stacked in Deployed status sp, _ := json.Marshal(d8updater.StatusPatch{ - Phase: v1alpha1.ModuleReleasePhaseSuperseded, + Phase: v1alpha1.DeckhouseReleasePhaseSuperseded, TransitionTime: metav1.NewTime(now), }) // everything except the last Deployed release @@ -110,7 +110,7 @@ func (r *deckhouseReleaseReconciler) cleanupDeckhouseRelease(ctx context.Context if len(deployedReleasesIndexes) > 0 && len(pendingReleasesIndexes) > 0 { lastDeployed := deployedReleasesIndexes[0] // releases are reversed, that's why we have to take the first one (latest Deployed release) sp, _ := json.Marshal(d8updater.StatusPatch{ - Phase: v1alpha1.ModuleReleasePhaseSkipped, + Phase: v1alpha1.DeckhouseReleasePhaseSkipped, Message: "Skipped by cleanup hook", TransitionTime: metav1.NewTime(now), }) diff --git a/deckhouse-controller/pkg/controller/deckhouse-release/client_test.go b/deckhouse-controller/pkg/controller/deckhouse-release/client_test.go index 959f1ddd5d..2ee2c0bd6a 100644 --- a/deckhouse-controller/pkg/controller/deckhouse-release/client_test.go +++ b/deckhouse-controller/pkg/controller/deckhouse-release/client_test.go @@ -37,11 +37,14 @@ import ( "sigs.k8s.io/yaml" "github.com/deckhouse/deckhouse/deckhouse-controller/pkg/apis/deckhouse.io/v1alpha1" + d8updater "github.com/deckhouse/deckhouse/deckhouse-controller/pkg/controller/deckhouse-release/updater" "github.com/deckhouse/deckhouse/deckhouse-controller/pkg/helpers" "github.com/deckhouse/deckhouse/go_lib/dependency" "github.com/deckhouse/deckhouse/pkg/log" ) +var testDeckhouseVersion = "v1.15.0" + func setupFakeController( t *testing.T, filename, values string, @@ -93,12 +96,14 @@ func setupControllerSettings( dc := dependency.NewDependencyContainer() rec := &deckhouseReleaseReconciler{ - client: cl, - dc: dc, - logger: log.NewNop(), - moduleManager: stubModulesManager{}, - updateSettings: helpers.NewDeckhouseSettingsContainer(ds), - metricStorage: metricstorage.NewMetricStorage(context.Background(), "", true, log.NewNop()), + client: cl, + deckhouseVersion: testDeckhouseVersion, + dc: dc, + logger: log.NewNop(), + moduleManager: stubModulesManager{}, + updateSettings: helpers.NewDeckhouseSettingsContainer(ds), + metricStorage: metricstorage.NewMetricStorage(context.Background(), "", true, log.NewNop()), + metricsUpdater: d8updater.NewMetricsUpdater(metricstorage.NewMetricStorage(context.Background(), "", true, log.NewNop())), } rec.clusterUUID = rec.getClusterUUID(context.Background()) diff --git a/deckhouse-controller/pkg/controller/deckhouse-release/controller.go b/deckhouse-controller/pkg/controller/deckhouse-release/controller.go index 6911240daf..280ea8d053 100644 --- a/deckhouse-controller/pkg/controller/deckhouse-release/controller.go +++ b/deckhouse-controller/pkg/controller/deckhouse-release/controller.go @@ -21,12 +21,15 @@ import ( "encoding/json" "errors" "fmt" + "log/slog" "net/http" - "os" + "strconv" "strings" "sync" "time" + "github.com/Masterminds/semver/v3" + aoapp "github.com/flant/addon-operator/pkg/app" metricstorage "github.com/flant/shell-operator/pkg/metric_storage" "github.com/gofrs/uuid/v5" gcr "github.com/google/go-containerregistry/pkg/name" @@ -43,6 +46,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" "github.com/deckhouse/deckhouse/deckhouse-controller/pkg/apis/deckhouse.io/v1alpha1" + "github.com/deckhouse/deckhouse/deckhouse-controller/pkg/controller/ctrlutils" d8updater "github.com/deckhouse/deckhouse/deckhouse-controller/pkg/controller/deckhouse-release/updater" "github.com/deckhouse/deckhouse/deckhouse-controller/pkg/controller/module-controllers/utils" "github.com/deckhouse/deckhouse/deckhouse-controller/pkg/helpers" @@ -55,6 +59,10 @@ import ( const ( metricReleasesGroup = "d8_releases" metricUpdatingGroup = "d8_updating" + + deckhouseNamespace = "d8-system" + deckhouseDeployment = "deckhouse" + deckhouseRegistrySecretName = "deckhouse-registry" ) const defaultCheckInterval = 15 * time.Second @@ -71,12 +79,22 @@ type deckhouseReleaseReconciler struct { preflightCountDown *sync.WaitGroup clusterUUID string releaseVersionImageHash string + + registrySecret *utils.DeckhouseRegistrySecret + metricsUpdater updater.MetricsUpdater + + deckhouseVersion string } func NewDeckhouseReleaseController(ctx context.Context, mgr manager.Manager, dc dependency.Container, moduleManager moduleManager, updateSettings *helpers.DeckhouseSettingsContainer, metricStorage *metricstorage.MetricStorage, - preflightCountDown *sync.WaitGroup, logger *log.Logger, + preflightCountDown *sync.WaitGroup, deckhouseVersion string, logger *log.Logger, ) error { + parsedVersion, err := semver.NewVersion(deckhouseVersion) + if err != nil { + return fmt.Errorf("parse deckhouse version: %w", err) + } + r := &deckhouseReleaseReconciler{ client: mgr.GetClient(), dc: dc, @@ -85,19 +103,21 @@ func NewDeckhouseReleaseController(ctx context.Context, mgr manager.Manager, dc updateSettings: updateSettings, metricStorage: metricStorage, preflightCountDown: preflightCountDown, + deckhouseVersion: fmt.Sprintf("v%d.%d.%d", parsedVersion.Major(), parsedVersion.Minor(), parsedVersion.Patch()), + + metricsUpdater: d8updater.NewMetricsUpdater(metricStorage), } // Add Preflight Check - err := mgr.Add(manager.RunnableFunc(r.PreflightCheck)) - if err != nil { - return err + if err = mgr.Add(manager.RunnableFunc(r.PreflightCheck)); err != nil { + return fmt.Errorf("add a runnable function: %w", err) } r.preflightCountDown.Add(1) // wait for cache sync go func() { if ok := mgr.GetCache().WaitForCacheSync(ctx); !ok { - r.logger.Fatalf("Sync cache failed") + r.logger.Fatal("Sync cache failed") } go r.updateByImageHashLoop(ctx) go r.checkDeckhouseReleaseLoop(ctx) @@ -123,13 +143,16 @@ func NewDeckhouseReleaseController(ctx context.Context, mgr manager.Manager, dc } func (r *deckhouseReleaseReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - var result ctrl.Result - r.logger.Debugf("%s release processing started", req.Name) - defer func() { r.logger.Debugf("%s release processing complete: %+v", req.Name, result) }() + var res ctrl.Result + + r.logger.Debug("release processing started", slog.String("resource_name", req.Name)) + defer func() { + r.logger.Debug("release processing complete", slog.String("resource_name", req.Name), slog.Any("reconcile_result", res)) + }() if r.updateSettings.Get().ReleaseChannel == "" { r.logger.Debug("release channel not set") - return result, nil + return res, nil } r.metricStorage.Grouped().ExpireGroupMetrics(metricReleasesGroup) @@ -137,19 +160,20 @@ func (r *deckhouseReleaseReconciler) Reconcile(ctx context.Context, req ctrl.Req release := new(v1alpha1.DeckhouseRelease) err := r.client.Get(ctx, req.NamespacedName, release) if err != nil { - r.logger.Debugf("get release: %s", err.Error()) // The DeckhouseRelease resource may no longer exist, in which case we stop // processing. if apierrors.IsNotFound(err) { - return result, nil + return res, nil } - return result, err + r.logger.Debug("get release", log.Err(err)) + + return res, err } if !release.DeletionTimestamp.IsZero() { - r.logger.Debugf("release deletion timestamp: %s", release.DeletionTimestamp.String()) - return result, nil + r.logger.Debug("release deletion", slog.String("deletion_timestamp", release.DeletionTimestamp.String())) + return res, nil } return r.createOrUpdateReconcile(ctx, release) @@ -167,7 +191,9 @@ func (r *deckhouseReleaseReconciler) getClusterUUID(ctx context.Context) string key := types.NamespacedName{Namespace: "d8-system", Name: "deckhouse-discovery"} err := r.client.Get(ctx, key, &secret) if err != nil { - r.logger.Warnf("Read clusterUUID from secret %s failed: %v. Generating random uuid", key, err) + r.logger.Warn("read clusterUUID from secret", slog.Any("namespaced_name", key), log.Err(err)) + r.logger.Warn("generating random uuid") + return uuid.Must(uuid.NewV4()).String() } @@ -179,199 +205,538 @@ func (r *deckhouseReleaseReconciler) getClusterUUID(ctx context.Context) string } func (r *deckhouseReleaseReconciler) createOrUpdateReconcile(ctx context.Context, dr *v1alpha1.DeckhouseRelease) (ctrl.Result, error) { - var result ctrl.Result + var res ctrl.Result // prepare releases switch dr.Status.Phase { // these phases should be ignored by predicate, but let's check it case "": + // set current restored release as deployed + if dr.GetCurrentRestored() { + return res, r.proceedRestoredRelease(ctx, dr) + } + // initial state - dr.Status.Phase = v1alpha1.ModuleReleasePhasePending + dr.Status.Phase = v1alpha1.DeckhouseReleasePhasePending dr.Status.TransitionTime = metav1.NewTime(r.dc.GetClock().Now().UTC()) - if e := r.client.Status().Update(ctx, dr); e != nil { - return ctrl.Result{Requeue: true}, e + if err := r.client.Status().Update(ctx, dr); err != nil { + return res, err } return ctrl.Result{Requeue: true}, nil // process to the next phase - case v1alpha1.ModuleReleasePhaseSkipped, v1alpha1.ModuleReleasePhaseSuperseded, v1alpha1.ModuleReleasePhaseSuspended: - r.logger.Debugf("release phase: %s", dr.Status.Phase) - return result, nil + case v1alpha1.DeckhouseReleasePhaseSkipped, v1alpha1.DeckhouseReleasePhaseSuperseded, v1alpha1.DeckhouseReleasePhaseSuspended: + r.logger.Debug("release phase", slog.String("phase", dr.Status.Phase)) + return res, nil - case v1alpha1.ModuleReleasePhaseDeployed: - // don't think we have to do anything with Deployed release - // probably, we have to move the Deployment's image update logic here + case v1alpha1.DeckhouseReleasePhaseDeployed: return r.reconcileDeployedRelease(ctx, dr) } // update pending release with suspend annotation - err := r.patchSuspendAnnotation(dr) + err := r.patchSuspendAnnotation(ctx, dr) if err != nil { - return ctrl.Result{RequeueAfter: defaultCheckInterval}, err + return res, err } - err = r.patchManualRelease(dr) + err = r.patchManualRelease(ctx, dr) if err != nil { - return ctrl.Result{RequeueAfter: defaultCheckInterval}, err + return res, err } return r.pendingReleaseReconcile(ctx, dr) } -func (r *deckhouseReleaseReconciler) patchManualRelease(dr *v1alpha1.DeckhouseRelease) error { +func (r *deckhouseReleaseReconciler) patchManualRelease(ctx context.Context, dr *v1alpha1.DeckhouseRelease) error { if r.updateSettings.Get().Update.Mode != updater.ModeManual.String() { return nil } - if !dr.GetManuallyApproved() { - dr.SetApprovedStatus(false) - // TODO: don't know yet how to count manual releases - // du.totalPendingManualReleases++ - } else { - dr.SetApprovedStatus(true) + drCopy := dr.DeepCopy() + + drCopy.SetApprovedStatus(drCopy.GetManuallyApproved()) + + err := r.client.Status().Patch(ctx, drCopy, client.MergeFrom(dr)) + if err != nil { + return fmt.Errorf("patch approved status: %w", err) } - return r.client.Status().Update(context.Background(), dr) + return nil +} + +func (r *deckhouseReleaseReconciler) proceedRestoredRelease(ctx context.Context, dr *v1alpha1.DeckhouseRelease) error { + dr.Status.Approved = true + dr.Status.Phase = v1alpha1.DeckhouseReleasePhaseDeployed + dr.Status.TransitionTime = metav1.NewTime(r.dc.GetClock().Now().UTC()) + dr.Status.Message = "Release object was restored" + + if err := r.client.Status().Update(ctx, dr); err != nil { + return err + } + + return nil } -func (r *deckhouseReleaseReconciler) patchSuspendAnnotation(dr *v1alpha1.DeckhouseRelease) error { +func (r *deckhouseReleaseReconciler) patchSuspendAnnotation(ctx context.Context, dr *v1alpha1.DeckhouseRelease) error { if !dr.GetSuspend() { return nil } - ctx := context.Background() - patch, _ := json.Marshal(map[string]any{ - "metadata": map[string]any{ - "annotations": map[string]any{ - "release.deckhouse.io/suspended": nil, - }, - }, - }) + drCopy := dr.DeepCopy() + + drCopy.Status.Phase = v1alpha1.DeckhouseReleasePhaseSuspended - p := client.RawPatch(types.MergePatchType, patch) - return r.client.Patch(ctx, dr, p) + delete(drCopy.Annotations, v1alpha1.DeckhouseReleaseAnnotationSuspended) + + err := r.client.Patch(ctx, drCopy, client.MergeFrom(dr)) + if err != nil { + return fmt.Errorf("patch suspend annotation: %w", err) + } + + err = r.client.Status().Patch(ctx, drCopy, client.MergeFrom(dr)) + if err != nil { + return fmt.Errorf("patch suspend phase: %w", err) + } + + return nil } +// pendingReleaseReconcile +// +// 1) Calculate task for current release +// 1.1) if skip - update phase to Skipped and stop reconcile +// 1.2) if await - update phase to Pending and requeue +// 1.3) if process - go forward +// 1.4) if forced - apply release +// 2) Apply if force with force logic +// 3) Check requirements +// 3.1) if not met any requirements - update phase to Pending with all requirements errors and requeue +// 4) Check deploy time and notify +// 5) Apply ussually release func (r *deckhouseReleaseReconciler) pendingReleaseReconcile(ctx context.Context, dr *v1alpha1.DeckhouseRelease) (ctrl.Result, error) { - var result ctrl.Result + var res ctrl.Result + + if r.registrySecret == nil { + // TODO: make registry service to check secrets in it (make issue) + registrySecret, err := r.getRegistrySecret(ctx) + if err != nil && !errors.Is(err, utils.ErrClusterIsBootstrappedFieldIsNotFound) { + return res, fmt.Errorf("get registry secret: %w", err) + } + + if err != nil { + r.registrySecret.ClusterIsBootstrapped = true + } + + r.registrySecret = registrySecret + } + + if r.isDeckhousePodReady(ctx) { + r.metricStorage.Grouped().ExpireGroupMetrics(metricUpdatingGroup) + } else { + r.metricStorage.Grouped().GaugeSet(metricUpdatingGroup, "d8_is_updating", 1, map[string]string{"releaseChannel": r.updateSettings.Get().ReleaseChannel}) + } + + oCalc := d8updater.NewTaskCalculator(r.client, r.logger) + task, err := oCalc.CalculatePendingReleaseOrder(ctx, dr) + if err != nil { + return res, err + } + + if dr.GetForce() { + r.logger.Warn("forced release found") - clusterBootstrapping := true - var imagesRegistry string - registrySecret, err := r.getRegistrySecret(ctx) - if apierrors.IsNotFound(err) { - err = nil + // deploy forced release without any checks (windows, requirements, approvals and so on) + err := r.ApplyRelease(ctx, dr, task) + if err != nil { + return res, fmt.Errorf("apply forced release: %w", err) + } + + // stop requeue because we restart deckhouse (deployment) + return ctrl.Result{}, nil } + + switch task.TaskType { + case d8updater.Process: + // pass + case d8updater.Skip: + err := r.updateReleaseStatus(ctx, dr, &v1alpha1.DeckhouseReleaseStatus{ + Phase: v1alpha1.DeckhouseReleasePhaseSkipped, + Message: task.Message, + }) + if err != nil { + r.logger.Warn("skip order status update ", slog.String("name", dr.GetName()), log.Err(err)) + return ctrl.Result{RequeueAfter: defaultCheckInterval}, nil + } + + return res, nil + case d8updater.Await: + err := r.updateReleaseStatus(ctx, dr, &v1alpha1.DeckhouseReleaseStatus{ + Phase: v1alpha1.DeckhouseReleasePhasePending, + Message: task.Message, + }) + if err != nil { + r.logger.Warn("await order status update ", slog.String("name", dr.GetName()), log.Err(err)) + } + + return ctrl.Result{RequeueAfter: defaultCheckInterval}, nil + } + + checker, err := d8updater.NewRequirementsChecker(r.client, r.moduleManager.GetEnabledModuleNames(), r.logger) if err != nil { - return result, fmt.Errorf("get registry secret: %w", err) + updateErr := r.updateReleaseStatus(ctx, dr, &v1alpha1.DeckhouseReleaseStatus{ + Phase: v1alpha1.DeckhouseReleasePhasePending, + Message: err.Error(), + }) + if updateErr != nil { + r.logger.Warn("create release checker status update ", slog.String("name", dr.GetName()), log.Err(err)) + } + + return ctrl.Result{RequeueAfter: defaultCheckInterval}, nil } - if registrySecret != nil { - clusterBootstrapped, ok := registrySecret.Data["clusterIsBootstrapped"] - if ok { - clusterBootstrapping = string(clusterBootstrapped) != `"true"` + reasons := checker.MetRequirements(dr) + if len(reasons) > 0 { + msgs := make([]string, 0, len(reasons)) + for _, reason := range reasons { + msgs = append(msgs, reason.Message) } - imagesRegistry = string(registrySecret.Data["imagesRegistry"]) + err := r.updateReleaseStatus(ctx, dr, &v1alpha1.DeckhouseReleaseStatus{ + Phase: v1alpha1.DeckhouseReleasePhasePending, + Message: strings.Join(msgs, ";"), + }) + if err != nil { + r.logger.Warn("met requirements status update ", slog.String("name", dr.GetName()), log.Err(err)) + } + + return ctrl.Result{RequeueAfter: defaultCheckInterval}, nil } - podReady := r.isDeckhousePodReady() + // TODO: it's maybe deprecated history about bootstrap deploying. delete??? + // + // if cluster needs bootstrap and we found only one release - apply release + if !r.registrySecret.ClusterIsBootstrapped && task.IsSingle { + err := r.ApplyRelease(ctx, dr, task) + if err != nil { + return res, fmt.Errorf("run single bootstrapping release deploy: %w", err) + } + + // stop requeue because we restart deckhouse (deployment) + return ctrl.Result{}, nil + } + + // handling error inside function + err = r.PreApplyReleaseCheck(ctx, dr, task) + if err != nil { + // ignore this err, just requeue because of check failed + return ctrl.Result{RequeueAfter: defaultCheckInterval}, nil + } + + err = r.ApplyRelease(ctx, dr, task) + if err != nil { + return res, fmt.Errorf("apply predicted release: %w", err) + } + + return ctrl.Result{RequeueAfter: defaultCheckInterval}, nil +} + +var ErrPreApplyCheckIsFailed = errors.New("pre apply check is failed") + +// PreApplyReleaseCheck checks final conditions before apply +// +// - Calculating deploy time (if zero - deploy) +func (r *deckhouseReleaseReconciler) PreApplyReleaseCheck(ctx context.Context, dr *v1alpha1.DeckhouseRelease, task *d8updater.Task) error { + metricLabels := updater.NewReleaseMetricLabels(dr) + + dtr := r.DeployTimeCalculate(ctx, dr, task, metricLabels) + + if metricLabels[updater.ManualApprovalRequired] == "true" { + metricLabels[updater.ReleaseQueueDepth] = strconv.Itoa(task.QueueDepth) + } + + // if the predicted release has an index less than the number of awaiting releases + // calculate and set releaseDepthQueue label + r.metricsUpdater.UpdateReleaseMetric(dr.GetName(), metricLabels) + + if dtr == nil { + return nil + } + + err := r.updateReleaseStatus(ctx, dr, &v1alpha1.DeckhouseReleaseStatus{ + Phase: v1alpha1.DeckhouseReleasePhasePending, + Message: dtr.Message, + }) + if err != nil { + r.logger.Warn("met release conditions status update ", slog.String("name", dr.GetName()), log.Err(err)) + } + + err = ctrlutils.UpdateWithRetry(ctx, r.client, dr, func() error { + if dr.Annotations == nil { + dr.Annotations = make(map[string]string, 2) + } + + dr.Annotations[v1alpha1.DeckhouseReleaseAnnotationIsUpdating] = "false" + dr.Annotations[v1alpha1.DeckhouseReleaseAnnotationNotified] = strconv.FormatBool(dtr.Notified) + + if !dtr.ReleaseApplyAfterTime.IsZero() { + dr.Spec.ApplyAfter = &metav1.Time{Time: dtr.ReleaseApplyAfterTime.UTC()} + + dr.Annotations[v1alpha1.DeckhouseReleaseAnnotationNotificationTimeShift] = "true" + } + + return nil + }) + if err != nil { + r.logger.Warn("met release conditions resource update ", slog.String("name", dr.GetName()), log.Err(err)) + } + + return ErrPreApplyCheckIsFailed +} + +const ( + msgReleaseIsBlockedByNotification = "Release is blocked, failed to send release notification" +) + +// DeployTimeCalculate calculate time for release deploy +// +// If patch, calculate by checking this conditions: +// - Canary +// - Notify +// - Window +// - ManualApproved +// +// If minor, calculate by checking this conditions: +// - Cooldown +// - Canary +// - Notify +// - Window +// - Manual Approved +func (r *deckhouseReleaseReconciler) DeployTimeCalculate(ctx context.Context, dr *v1alpha1.DeckhouseRelease, task *d8updater.Task, metricLabels updater.MetricLabels) *d8updater.DeployTimeReason { us := r.updateSettings.Get() + dus := &updater.Settings{ NotificationConfig: us.Update.NotificationConfig, DisruptionApprovalMode: us.Update.DisruptionApprovalMode, - Mode: updater.ParseUpdateMode(us.Update.Mode), - Windows: us.Update.Windows, + // if we have wrong mode - autopatch + Mode: updater.ParseUpdateMode(us.Update.Mode), + Windows: us.Update.Windows, } - releaseData := getReleaseData(dr) - deckhouseUpdater := d8updater.NewDeckhouseUpdater( - ctx, r.logger, r.client, r.dc, dus, releaseData, r.metricStorage, podReady, - clusterBootstrapping, imagesRegistry, r.moduleManager.GetEnabledModuleNames(), - ) - if podReady { - r.metricStorage.Grouped().ExpireGroupMetrics(metricUpdatingGroup) + releaseNotifier := d8updater.NewReleaseNotifier(dus) + timeChecker := d8updater.NewDeployTimeChecker(r.dc, dus, r.isDeckhousePodReady, r.logger) + + var deployTimeResult *d8updater.DeployTimeResult - if releaseData.IsUpdating { - _ = deckhouseUpdater.ChangeUpdatingFlag(false) + if task.IsPatch { + deployTimeResult = timeChecker.CalculatePatchDeployTime(dr, metricLabels) + + notifyErr := releaseNotifier.SendPatchReleaseNotification(ctx, dr, deployTimeResult.ReleaseApplyAfterTime, metricLabels) + if notifyErr != nil { + r.logger.Warn("send [patch] release notification", log.Err(notifyErr)) + + return &d8updater.DeployTimeReason{ + Message: msgReleaseIsBlockedByNotification, + ReleaseApplyAfterTime: deployTimeResult.ReleaseApplyAfterTime, + } } - } else if releaseData.IsUpdating { - r.metricStorage.Grouped().GaugeSet(metricUpdatingGroup, "d8_is_updating", 1, map[string]string{"releaseChannel": r.updateSettings.Get().ReleaseChannel}) + + dtr := timeChecker.ProcessPatchReleaseDeployTime(dr, deployTimeResult) + if dtr != nil { + dtr.Notified = true + } + + return dtr } - { - var releases v1alpha1.DeckhouseReleaseList - err = r.client.List(ctx, &releases) - if err != nil { - return result, fmt.Errorf("get deckhouse releases: %w", err) + + // for minor release we must check additional conditions + checker := d8updater.NewPreApplyChecker(dus, r.logger) + reasons := checker.MetRequirements(dr) + if len(reasons) > 0 { + metricLabels.SetTrue(updater.DisruptionApprovalRequired) + + msgs := make([]string, 0, len(reasons)) + for _, reason := range reasons { + msgs = append(msgs, reason.Message) } - pointerReleases := make([]*v1alpha1.DeckhouseRelease, 0, len(releases.Items)) - for _, rl := range releases.Items { - pointerReleases = append(pointerReleases, &rl) + return &d8updater.DeployTimeReason{ + Message: fmt.Sprintf("release blocked, disruption approval required: %s", strings.Join(msgs, ", ")), } - deckhouseUpdater.SetReleases(pointerReleases) } - if deckhouseUpdater.ReleasesCount() == 0 { - r.logger.Debug("releases count is zero") - return result, nil + deployTimeResult = timeChecker.CalculateMinorDeployTime(dr, metricLabels) + + notifyErr := releaseNotifier.SendMinorReleaseNotification(ctx, dr, deployTimeResult.ReleaseApplyAfterTime, metricLabels) + if notifyErr != nil { + r.logger.Warn("send minor release notification", log.Err(notifyErr)) + + return &d8updater.DeployTimeReason{ + Message: msgReleaseIsBlockedByNotification, + ReleaseApplyAfterTime: deployTimeResult.ReleaseApplyAfterTime, + } } - // predict next patch for Deploy - deckhouseUpdater.PredictNextRelease(dr) + dtr := timeChecker.ProcessMinorReleaseDeployTime(ctx, dr, deployTimeResult, task.DeployedReleaseInfo) + if dtr != nil { + dtr.Notified = true + } + + return dtr +} + +// ApplyRelease applies predicted release +func (r *deckhouseReleaseReconciler) ApplyRelease(ctx context.Context, dr *v1alpha1.DeckhouseRelease, task *d8updater.Task) error { + var dri *d8updater.ReleaseInfo - // has already Deployed the latest release - if deckhouseUpdater.LastReleaseDeployed() { - r.logger.Debug("latest release is deployed") - return result, nil + if task != nil { + dri = task.DeployedReleaseInfo } - // set skipped releases to PhaseSkipped - if err = deckhouseUpdater.CommitSkippedReleases(); err != nil { - return result, err + err := r.runReleaseDeploy(ctx, dr, dri) + if err != nil { + return fmt.Errorf("run release deploy: %w", err) } - if rel := deckhouseUpdater.GetPredictedRelease(); rel != nil { - if rel.GetName() != dr.GetName() { - // requeue all releases to keep syncing releases' metrics - r.logger.Debugf("processing wrong release (current: %s, predicted: %s)", dr.Name, rel.Name) - return ctrl.Result{RequeueAfter: defaultCheckInterval}, nil + return nil +} + +// runReleaseDeploy +// +// 1) bump deckhouse deployment (retry if error) if the dryrun annotations isn't set (stop deploying in the opposite case) +// 2) bump previous deployment status superseded (retry if error) +// 3) bump release annotations (retry if error) +// 3) bump release status to deployed (retry if error) +func (r *deckhouseReleaseReconciler) runReleaseDeploy(ctx context.Context, dr *v1alpha1.DeckhouseRelease, deployedReleaseInfo *d8updater.ReleaseInfo) error { + r.logger.Info("applying release", slog.String("name", dr.GetName())) + + err := r.bumpDeckhouseDeployment(ctx, dr) + if err != nil { + return fmt.Errorf("deploy release: %w", err) + } + + if deployedReleaseInfo != nil { + err := r.updateReleaseStatus(ctx, newDeckhouseReleaseWithName(deployedReleaseInfo.Name), &v1alpha1.DeckhouseReleaseStatus{ + Phase: v1alpha1.DeckhouseReleasePhaseSuperseded, + Message: "", + }) + if err != nil { + r.logger.Error("update status", slog.String("release", deployedReleaseInfo.Name), log.Err(err)) } } - // some release is forced, burn everything, apply this patch! - if deckhouseUpdater.HasForceRelease() { - err = deckhouseUpdater.ApplyForcedRelease(ctx) - if err == nil { - return result, nil + err = ctrlutils.UpdateWithRetry(ctx, r.client, dr, func() error { + annotations := map[string]string{ + v1alpha1.DeckhouseReleaseAnnotationIsUpdating: "true", + v1alpha1.DeckhouseReleaseAnnotationNotified: "false", } - return result, fmt.Errorf("apply forced release: %w", err) + if dr.Annotations == nil { + dr.Annotations = make(map[string]string, 2) + } + + for k, v := range annotations { + dr.Annotations[k] = v + } + + if dr.GetApplyNow() { + delete(dr.Annotations, v1alpha1.DeckhouseReleaseAnnotationApplyNow) + } + + if dr.GetForce() { + delete(dr.Annotations, v1alpha1.DeckhouseReleaseAnnotationForce) + } + + return nil + }) + if err != nil { + return fmt.Errorf("update with retry: %w", err) } - err = deckhouseUpdater.ApplyPredictedRelease() + err = r.updateReleaseStatus(ctx, dr, &v1alpha1.DeckhouseReleaseStatus{ + Phase: v1alpha1.DeckhouseReleasePhaseDeployed, + }) if err != nil { - return r.wrapApplyReleaseError(err) + return fmt.Errorf("update status with retry: %w", err) } - return ctrl.Result{RequeueAfter: defaultCheckInterval}, nil + return nil } -func (r *deckhouseReleaseReconciler) wrapApplyReleaseError(err error) (ctrl.Result, error) { - var result ctrl.Result - var notReadyErr *updater.NotReadyForDeployError - if errors.As(err, ¬ReadyErr) { - r.logger.Info(err.Error()) - // TODO: requeue all releases if deckhouse update settings is changed - // requeueAfter := notReadyErr.RetryDelay() - // if requeueAfter == 0 { - // requeueAfter = defaultCheckInterval - // } - // r.logger.Infof("%s: retry after %s", err.Error(), requeueAfter) - // return ctrl.Result{RequeueAfter: requeueAfter}, nil - return ctrl.Result{RequeueAfter: defaultCheckInterval}, nil +var ErrDeploymentContainerIsNotFound = errors.New("deployment container is not found") + +func (r *deckhouseReleaseReconciler) bumpDeckhouseDeployment(ctx context.Context, dr *v1alpha1.DeckhouseRelease) error { + key := client.ObjectKey{Namespace: deckhouseNamespace, Name: deckhouseDeployment} + + depl := new(appsv1.Deployment) + + err := r.client.Get(ctx, key, depl) + if err != nil { + return fmt.Errorf("get deployment %s: %w", key, err) + } + + // dryrun for testing purpose + if dr.GetDryRun() { + go func() { + r.logger.Debug("dryrun start soon...") + + time.Sleep(3 * time.Second) + + r.logger.Debug("dryrun started") + + // because we do not know how long is parent context and how long will be update + // 1 minute - magic constant + ctxwt, cancel := context.WithTimeout(context.Background(), 1*time.Minute) + defer cancel() + + releases := new(v1alpha1.DeckhouseReleaseList) + err = r.client.List(ctxwt, releases) + if err != nil { + r.logger.Error("dryrun list deckhouse releases", log.Err(err)) + + return + } + + for _, release := range releases.Items { + release := &release + + if release.GetName() == dr.GetName() { + continue + } + + if release.Status.Phase != v1alpha1.DeckhouseReleasePhasePending { + continue + } + + // update releases to trigger their requeue + err := ctrlutils.UpdateWithRetry(ctxwt, r.client, release, func() error { + if release.Annotations == nil { + release.Annotations = make(map[string]string, 1) + } + + release.Annotations[v1alpha1.DeckhouseReleaseAnnotationTriggeredByDryrun] = dr.GetName() + + return nil + }) + if err != nil { + r.logger.Error("dryrun update release to requeue", log.Err(err)) + } + + r.logger.Debug("dryrun release successfully updated", slog.String("release", release.Name)) + } + }() + + return nil } - return result, fmt.Errorf("apply predicted release: %w", err) + return ctrlutils.UpdateWithRetry(ctx, r.client, depl, func() error { + if len(depl.Spec.Template.Spec.Containers) == 0 { + return ErrDeploymentContainerIsNotFound + } + + depl.Spec.Template.Spec.Containers[0].Image = r.registrySecret.ImageRegistry + ":" + dr.Spec.Version + + return nil + }) } func (r *deckhouseReleaseReconciler) getDeckhouseLatestPod(ctx context.Context) (*corev1.Pod, error) { @@ -447,21 +812,22 @@ func (r *deckhouseReleaseReconciler) tagUpdate(ctx context.Context, leaderPod *c repo := imageRepoTag.Context().Name() tag := imageRepoTag.TagStr() - registrySecret, err := r.getRegistrySecret(ctx) - if apierrors.IsNotFound(err) { - err = nil - } - if err != nil { - return err + if r.registrySecret == nil { + // TODO: make registry service to check secrets in it (make issue) + registrySecret, err := r.getRegistrySecret(ctx) + if client.IgnoreNotFound(err) != nil { + return err + } + + r.registrySecret = registrySecret } var opts []cr.Option - if registrySecret != nil { - drs, _ := utils.ParseDeckhouseRegistrySecret(registrySecret.Data) + if r.registrySecret != nil { rconf := &utils.RegistryConfig{ - DockerConfig: drs.DockerConfig, - Scheme: drs.Scheme, - CA: drs.CA, + DockerConfig: r.registrySecret.DockerConfig, + Scheme: r.registrySecret.Scheme, + CA: r.registrySecret.CA, UserAgent: r.clusterUUID, } opts = utils.GenerateRegistryOptions(rconf, r.logger) @@ -520,24 +886,31 @@ func (r *deckhouseReleaseReconciler) tagUpdate(ctx context.Context, leaderPod *c return nil } -func (r *deckhouseReleaseReconciler) getRegistrySecret(ctx context.Context) (*corev1.Secret, error) { - var secret corev1.Secret - key := types.NamespacedName{Namespace: "d8-system", Name: "deckhouse-registry"} - err := r.client.Get(ctx, key, &secret) +func (r *deckhouseReleaseReconciler) getRegistrySecret(ctx context.Context) (*utils.DeckhouseRegistrySecret, error) { + key := types.NamespacedName{Namespace: deckhouseNamespace, Name: deckhouseRegistrySecretName} + + secret := new(corev1.Secret) + + err := r.client.Get(ctx, key, secret) if err != nil { return nil, fmt.Errorf("get secret %s: %w", key, err) } - return &secret, nil + regSecret, err := utils.ParseDeckhouseRegistrySecret(secret.Data) + if errors.Is(err, utils.ErrImageRegistryFieldIsNotFound) { + regSecret.ImageRegistry = regSecret.Address + regSecret.Path + } + + return regSecret, nil } -func (r *deckhouseReleaseReconciler) isDeckhousePodReady() bool { - deckhousePodIP := os.Getenv("ADDON_OPERATOR_LISTEN_ADDRESS") +func (r *deckhouseReleaseReconciler) isDeckhousePodReady(ctx context.Context) bool { + deckhousePodIP := aoapp.ListenAddress url := fmt.Sprintf("http://%s:4222/readyz", deckhousePodIP) - req, err := http.NewRequest(http.MethodGet, url, nil) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { - r.logger.Errorf("error getting deckhouse pod readyz status: %s", err) + r.logger.Error("error getting deckhouse pod readyz", log.Err(err)) } resp, err := r.dc.GetHTTPClient().Do(req) @@ -557,40 +930,59 @@ func (r *deckhouseReleaseReconciler) updateByImageHashLoop(ctx context.Context) deckhouseLeaderPod, err := r.getDeckhouseLatestPod(ctx) if err != nil { - r.logger.Warnf("Error getting deckhouse pods: %s", err) + r.logger.Warn("getting deckhouse pods", log.Err(err)) return } if deckhouseLeaderPod == nil { - r.logger.Debug("Deckhouse pods not found. Skipping update") + r.logger.Debug("deckhouse pods not found. Skipping update") return } err = r.tagUpdate(ctx, deckhouseLeaderPod) if err != nil { - r.logger.Errorf("deckhouse image tag update failed: %s", err) + r.logger.Error("deckhouse image tag update", log.Err(err)) } }, 15*time.Second) } func (r *deckhouseReleaseReconciler) reconcileDeployedRelease(ctx context.Context, dr *v1alpha1.DeckhouseRelease) (ctrl.Result, error) { - var result ctrl.Result + var res ctrl.Result + + if r.isDeckhousePodReady(ctx) { + err := ctrlutils.UpdateWithRetry(ctx, r.client, dr, func() error { + if dr.Annotations == nil { + dr.Annotations = make(map[string]string, 2) + } - if r.isDeckhousePodReady() { - data := getReleaseData(dr) - data.IsUpdating = false - err := r.newUpdaterKubeAPI().SaveReleaseData(ctx, dr, data) + dr.Annotations[v1alpha1.DeckhouseReleaseAnnotationIsUpdating] = "false" + dr.Annotations[v1alpha1.DeckhouseReleaseAnnotationNotified] = "true" + + return nil + }) if err != nil { - return result, fmt.Errorf("change updating flag: %w", err) + return res, err } - return result, nil + + return res, nil } return ctrl.Result{RequeueAfter: defaultCheckInterval}, nil } -func (r *deckhouseReleaseReconciler) newUpdaterKubeAPI() *d8updater.KubeAPI { - return d8updater.NewKubeAPI(r.client, r.dc, "") +func (r *deckhouseReleaseReconciler) updateReleaseStatus(ctx context.Context, dr *v1alpha1.DeckhouseRelease, status *v1alpha1.DeckhouseReleaseStatus) error { + r.logger.Debug("refresh release status", slog.String("name", dr.GetName())) + + return ctrlutils.UpdateStatusWithRetry(ctx, r.client, dr, func() error { + if dr.Status.Phase != status.Phase { + dr.Status.TransitionTime = metav1.NewTime(r.dc.GetClock().Now().UTC()) + } + + dr.Status.Phase = status.Phase + dr.Status.Message = status.Message + + return nil + }) } func getDeckhouseContainerIndex(containers []corev1.Container) int { @@ -613,9 +1005,10 @@ func getDeckhouseContainerStatusIndex(statuses []corev1.ContainerStatus) int { return -1 } -func getReleaseData(dr *v1alpha1.DeckhouseRelease) updater.DeckhouseReleaseData { - return updater.DeckhouseReleaseData{ - IsUpdating: dr.Annotations[d8updater.IsUpdatingAnnotation] == "true", - Notified: dr.Annotations[d8updater.NotifiedAnnotation] == "true", +func newDeckhouseReleaseWithName(name string) *v1alpha1.DeckhouseRelease { + return &v1alpha1.DeckhouseRelease{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, } } diff --git a/deckhouse-controller/pkg/controller/deckhouse-release/controller_test.go b/deckhouse-controller/pkg/controller/deckhouse-release/controller_test.go index d11193ed70..d4bf81e5be 100644 --- a/deckhouse-controller/pkg/controller/deckhouse-release/controller_test.go +++ b/deckhouse-controller/pkg/controller/deckhouse-release/controller_test.go @@ -240,9 +240,12 @@ func (suite *ControllerTestSuite) TestCreateReconcile() { mup.Update.Windows = update.Windows{{From: "8:00", To: "8:01"}} suite.setupController("patch-out-of-update-window.yaml", initValues, mup) - dr := suite.getDeckhouseRelease("v1.25.1") + dr := suite.getDeckhouseRelease("v1.26.0") _, err := suite.ctr.createOrUpdateReconcile(ctx, dr) require.NoError(suite.T(), err) + dr = suite.getDeckhouseRelease("v1.25.1") + _, err = suite.ctr.createOrUpdateReconcile(ctx, dr) + require.NoError(suite.T(), err) }) suite.Run("Deckhouse previous release is not ready", func() { @@ -286,9 +289,12 @@ func (suite *ControllerTestSuite) TestCreateReconcile() { mup.Update.Mode = updater.ModeManual.String() suite.setupController("auto-deploy-patch-release-in-manual-mode.yaml", initValues, mup) - dr := suite.getDeckhouseRelease("v1.25.1") + dr := suite.getDeckhouseRelease("v1.26.0") _, err := suite.ctr.createOrUpdateReconcile(ctx, dr) require.NoError(suite.T(), err) + dr = suite.getDeckhouseRelease("v1.25.1") + _, err = suite.ctr.createOrUpdateReconcile(ctx, dr) + require.NoError(suite.T(), err) }) suite.Run("Manual approval mode with canary process", func() { @@ -355,10 +361,114 @@ func (suite *ControllerTestSuite) TestCreateReconcile() { }) suite.Run("Few patch releases", func() { + dependency.TestDC.HTTPClient.DoMock. + Expect(&http.Request{}). + Return(&http.Response{ + StatusCode: http.StatusInternalServerError, + }, errors.New("some internal error")) + suite.setupController("few-patch-releases.yaml", initValues, embeddedMUP) - dr := suite.getDeckhouseRelease("v1.32.0") + dr := suite.getDeckhouseRelease("v1.31.1") + _, err := suite.ctr.createOrUpdateReconcile(ctx, dr) + require.NoError(suite.T(), err) + dr = suite.getDeckhouseRelease("v1.31.2") + _, err = suite.ctr.createOrUpdateReconcile(ctx, dr) + require.NoError(suite.T(), err) + dr = suite.getDeckhouseRelease("v1.31.3") + _, err = suite.ctr.createOrUpdateReconcile(ctx, dr) + require.NoError(suite.T(), err) + dr = suite.getDeckhouseRelease("v1.32.0") + _, err = suite.ctr.createOrUpdateReconcile(ctx, dr) + require.NoError(suite.T(), err) + }) + + suite.Run("few minor releases", func() { + dependency.TestDC.HTTPClient.DoMock. + Expect(&http.Request{}). + Return(&http.Response{ + StatusCode: http.StatusOK, + }, nil) + + suite.setupController("few-minor-releases.yaml", initValues, embeddedMUP) + dr := suite.getDeckhouseRelease("v1.31.0") + _, err := suite.ctr.createOrUpdateReconcile(ctx, dr) + require.NoError(suite.T(), err) + dr = suite.getDeckhouseRelease("v1.32.0") + _, err = suite.ctr.createOrUpdateReconcile(ctx, dr) + require.NoError(suite.T(), err) + + dependency.TestDC.HTTPClient.DoMock. + Expect(&http.Request{}). + Return(&http.Response{ + StatusCode: http.StatusInternalServerError, + }, errors.New("some internal error")) + + dr = suite.getDeckhouseRelease("v1.33.0") + _, err = suite.ctr.createOrUpdateReconcile(ctx, dr) + require.NoError(suite.T(), err) + dr = suite.getDeckhouseRelease("v1.34.0") + _, err = suite.ctr.createOrUpdateReconcile(ctx, dr) + require.NoError(suite.T(), err) + dr = suite.getDeckhouseRelease("v1.35.0") + _, err = suite.ctr.createOrUpdateReconcile(ctx, dr) + require.NoError(suite.T(), err) + }) + + suite.Run("few minor releases with version more than one from deployed", func() { + dependency.TestDC.HTTPClient.DoMock. + Expect(&http.Request{}). + Return(&http.Response{ + StatusCode: http.StatusOK, + }, nil) + + suite.setupController("few-minor-releases-version-more-than-one.yaml", initValues, embeddedMUP) + dr := suite.getDeckhouseRelease("v1.33.0") _, err := suite.ctr.createOrUpdateReconcile(ctx, dr) require.NoError(suite.T(), err) + dr = suite.getDeckhouseRelease("v1.34.0") + _, err = suite.ctr.createOrUpdateReconcile(ctx, dr) + require.NoError(suite.T(), err) + + dependency.TestDC.HTTPClient.DoMock. + Expect(&http.Request{}). + Return(&http.Response{ + StatusCode: http.StatusInternalServerError, + }, errors.New("some internal error")) + + dr = suite.getDeckhouseRelease("v1.33.0") + _, err = suite.ctr.createOrUpdateReconcile(ctx, dr) + require.NoError(suite.T(), err) + dr = suite.getDeckhouseRelease("v1.34.0") + _, err = suite.ctr.createOrUpdateReconcile(ctx, dr) + require.NoError(suite.T(), err) + dr = suite.getDeckhouseRelease("v1.35.0") + _, err = suite.ctr.createOrUpdateReconcile(ctx, dr) + require.NoError(suite.T(), err) + }) + + suite.Run("forced through few minor releases", func() { + dependency.TestDC.HTTPClient.DoMock. + Expect(&http.Request{}). + Return(&http.Response{ + StatusCode: http.StatusInternalServerError, + }, errors.New("some internal error")) + + suite.setupController("forced-few-minor-releases.yaml", initValues, embeddedMUP) + dr := suite.getDeckhouseRelease("v1.31.0") + _, err := suite.ctr.createOrUpdateReconcile(ctx, dr) + require.NoError(suite.T(), err) + dr = suite.getDeckhouseRelease("v1.32.0") + _, err = suite.ctr.createOrUpdateReconcile(ctx, dr) + require.NoError(suite.T(), err) + dr = suite.getDeckhouseRelease("v1.33.0") + _, err = suite.ctr.createOrUpdateReconcile(ctx, dr) + require.NoError(suite.T(), err) + dr = suite.getDeckhouseRelease("v1.34.0") + _, err = suite.ctr.createOrUpdateReconcile(ctx, dr) + require.NoError(suite.T(), err) + dr = suite.getDeckhouseRelease("v1.35.0") + _, err = suite.ctr.createOrUpdateReconcile(ctx, dr) + require.NoError(suite.T(), err) }) suite.Run("Pending Manual release on cluster bootstrap", func() { @@ -376,9 +486,12 @@ func (suite *ControllerTestSuite) TestCreateReconcile() { suite.Run("Forced release", func() { suite.setupController("forced-release.yaml", initValues, embeddedMUP) - dr := suite.getDeckhouseRelease("v1.31.1") + dr := suite.getDeckhouseRelease("v1.31.0") _, err := suite.ctr.createOrUpdateReconcile(ctx, dr) require.NoError(suite.T(), err) + dr = suite.getDeckhouseRelease("v1.31.1") + _, err = suite.ctr.createOrUpdateReconcile(ctx, dr) + require.NoError(suite.T(), err) }) suite.Run("Postponed release", func() { @@ -397,9 +510,12 @@ func (suite *ControllerTestSuite) TestCreateReconcile() { suite.Run("Suspend release", func() { suite.setupController("suspend-release.yaml", initValues, embeddedMUP) - dr := suite.getDeckhouseRelease("v1.25.2") + dr := suite.getDeckhouseRelease("v1.25.1") _, err := suite.ctr.createOrUpdateReconcile(ctx, dr) require.NoError(suite.T(), err) + dr = suite.getDeckhouseRelease("v1.25.2") + _, err = suite.ctr.createOrUpdateReconcile(ctx, dr) + require.NoError(suite.T(), err) }) suite.Run("Release with not met requirements", func() { @@ -746,9 +862,12 @@ func (suite *ControllerTestSuite) TestCreateReconcile() { }, nil) suite.setupController("auto-patch-mode.yaml", initValues, mup) - dr := suite.getDeckhouseRelease("v1.26.3") + dr := suite.getDeckhouseRelease("v1.26.2") _, err := suite.ctr.createOrUpdateReconcile(ctx, dr) require.NoError(suite.T(), err) + dr = suite.getDeckhouseRelease("v1.26.3") + _, err = suite.ctr.createOrUpdateReconcile(ctx, dr) + require.NoError(suite.T(), err) }) suite.Run("Test autoPatch-mode for postponed minor release", func() { diff --git a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/few-minor-releases-version-more-than-one.yaml b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/few-minor-releases-version-more-than-one.yaml new file mode 100644 index 0000000000..59b11cafd5 --- /dev/null +++ b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/few-minor-releases-version-more-than-one.yaml @@ -0,0 +1,66 @@ +--- +apiVersion: v1 +kind: Pod +metadata: + name: deckhouse-6f46df5bd7-nk4j7 + namespace: d8-system + labels: + app: deckhouse +spec: + containers: + - name: deckhouse + image: dev-registry.deckhouse.io/sys/deckhouse-oss:v1.2.3 +status: + containerStatuses: + - containerID: containerd://9990d3eccb8657d0bfe755672308831b6d0fab7f3aac553487c60bf0f076b2e3 + imageID: dev-registry.deckhouse.io/sys/deckhouse-oss/dev@sha256:d57f01a88e54f863ff5365c989cb4e2654398fa274d46389e0af749090b862d1 + ready: true +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: deckhouse + namespace: d8-system +spec: + template: + spec: + containers: + - name: deckhouse + image: my.registry.com/deckhouse:v1.25.0 + +--- +apiVersion: deckhouse.io/v1alpha1 +kind: DeckhouseRelease +metadata: + name: v1.31.0 +spec: + version: "v1.31.0" +status: + phase: Deployed +--- +apiVersion: deckhouse.io/v1alpha1 +kind: DeckhouseRelease +metadata: + name: v1.33.0 +spec: + version: "v1.33.0" +status: + phase: Pending +--- +apiVersion: deckhouse.io/v1alpha1 +kind: DeckhouseRelease +metadata: + name: v1.34.0 +spec: + version: "v1.34.0" +status: + phase: Pending +--- +apiVersion: deckhouse.io/v1alpha1 +kind: DeckhouseRelease +metadata: + name: v1.35.0 +spec: + version: "v1.35.0" +status: + phase: Pending diff --git a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/few-minor-releases.yaml b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/few-minor-releases.yaml new file mode 100644 index 0000000000..35697956c9 --- /dev/null +++ b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/few-minor-releases.yaml @@ -0,0 +1,75 @@ +--- +apiVersion: v1 +kind: Pod +metadata: + name: deckhouse-6f46df5bd7-nk4j7 + namespace: d8-system + labels: + app: deckhouse +spec: + containers: + - name: deckhouse + image: dev-registry.deckhouse.io/sys/deckhouse-oss:v1.2.3 +status: + containerStatuses: + - containerID: containerd://9990d3eccb8657d0bfe755672308831b6d0fab7f3aac553487c60bf0f076b2e3 + imageID: dev-registry.deckhouse.io/sys/deckhouse-oss/dev@sha256:d57f01a88e54f863ff5365c989cb4e2654398fa274d46389e0af749090b862d1 + ready: true +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: deckhouse + namespace: d8-system +spec: + template: + spec: + containers: + - name: deckhouse + image: my.registry.com/deckhouse:v1.25.0 + +--- +apiVersion: deckhouse.io/v1alpha1 +kind: DeckhouseRelease +metadata: + name: v1.31.0 +spec: + version: "v1.31.0" +status: + phase: Deployed +--- +apiVersion: deckhouse.io/v1alpha1 +kind: DeckhouseRelease +metadata: + name: v1.32.0 +spec: + version: "v1.32.0" +status: + phase: Pending +--- +apiVersion: deckhouse.io/v1alpha1 +kind: DeckhouseRelease +metadata: + name: v1.33.0 +spec: + version: "v1.33.0" +status: + phase: Pending +--- +apiVersion: deckhouse.io/v1alpha1 +kind: DeckhouseRelease +metadata: + name: v1.34.0 +spec: + version: "v1.34.0" +status: + phase: Pending +--- +apiVersion: deckhouse.io/v1alpha1 +kind: DeckhouseRelease +metadata: + name: v1.35.0 +spec: + version: "v1.35.0" +status: + phase: Pending diff --git a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/forced-few-minor-releases.yaml b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/forced-few-minor-releases.yaml new file mode 100644 index 0000000000..cf7cabbaa8 --- /dev/null +++ b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/forced-few-minor-releases.yaml @@ -0,0 +1,77 @@ +--- +apiVersion: v1 +kind: Pod +metadata: + name: deckhouse-6f46df5bd7-nk4j7 + namespace: d8-system + labels: + app: deckhouse +spec: + containers: + - name: deckhouse + image: dev-registry.deckhouse.io/sys/deckhouse-oss:v1.2.3 +status: + containerStatuses: + - containerID: containerd://9990d3eccb8657d0bfe755672308831b6d0fab7f3aac553487c60bf0f076b2e3 + imageID: dev-registry.deckhouse.io/sys/deckhouse-oss/dev@sha256:d57f01a88e54f863ff5365c989cb4e2654398fa274d46389e0af749090b862d1 + ready: true +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: deckhouse + namespace: d8-system +spec: + template: + spec: + containers: + - name: deckhouse + image: my.registry.com/deckhouse:v1.25.0 + +--- +apiVersion: deckhouse.io/v1alpha1 +kind: DeckhouseRelease +metadata: + name: v1.31.0 +spec: + version: "v1.31.0" +status: + phase: Deployed +--- +apiVersion: deckhouse.io/v1alpha1 +kind: DeckhouseRelease +metadata: + name: v1.32.0 +spec: + version: "v1.32.0" +status: + phase: Pending +--- +apiVersion: deckhouse.io/v1alpha1 +kind: DeckhouseRelease +metadata: + name: v1.33.0 +spec: + version: "v1.33.0" +status: + phase: Pending +--- +apiVersion: deckhouse.io/v1alpha1 +kind: DeckhouseRelease +metadata: + name: v1.34.0 +spec: + version: "v1.34.0" +status: + phase: Pending +--- +apiVersion: deckhouse.io/v1alpha1 +kind: DeckhouseRelease +metadata: + name: v1.35.0 + annotations: + release.deckhouse.io/force: true +spec: + version: "v1.35.0" +status: + phase: Pending diff --git a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/applied-now-postponed-release.yaml b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/applied-now-postponed-release.yaml index e0362dd901..ca13b21965 100644 --- a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/applied-now-postponed-release.yaml +++ b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/applied-now-postponed-release.yaml @@ -23,7 +23,7 @@ metadata: release.deckhouse.io/notified: "false" creationTimestamp: null name: v1.25.1 - resourceVersion: "1002" + resourceVersion: "1001" spec: applyAfter: "2222-11-11T23:23:23Z" version: v1.25.1 diff --git a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/apply-now-autopatch-mode-is-set.yaml b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/apply-now-autopatch-mode-is-set.yaml index 4a7a6de50c..3ce21b3feb 100644 --- a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/apply-now-autopatch-mode-is-set.yaml +++ b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/apply-now-autopatch-mode-is-set.yaml @@ -23,7 +23,7 @@ metadata: release.deckhouse.io/notified: "false" creationTimestamp: null name: v1.26.0 - resourceVersion: "1002" + resourceVersion: "1001" spec: version: v1.26.0 status: diff --git a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/apply-now-deckhouse-previous-release-is-not-ready.yaml b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/apply-now-deckhouse-previous-release-is-not-ready.yaml index fa1e5e147b..6533aef6bc 100644 --- a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/apply-now-deckhouse-previous-release-is-not-ready.yaml +++ b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/apply-now-deckhouse-previous-release-is-not-ready.yaml @@ -20,14 +20,16 @@ kind: DeckhouseRelease metadata: annotations: release.deckhouse.io/apply-now: "true" + release.deckhouse.io/isUpdating: "false" + release.deckhouse.io/notified: "true" creationTimestamp: null name: v1.26.0 - resourceVersion: "1000" + resourceVersion: "1001" spec: version: v1.26.0 status: approved: false - message: Awaiting for Deckhouse pod to be ready + message: awaiting for Deckhouse v1.25.0 pod to be ready phase: Pending transitionTime: null --- diff --git a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/apply-now-manual-approval-mode-is-set.yaml b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/apply-now-manual-approval-mode-is-set.yaml index 111e63e1ba..4a7a6de50c 100644 --- a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/apply-now-manual-approval-mode-is-set.yaml +++ b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/apply-now-manual-approval-mode-is-set.yaml @@ -23,7 +23,7 @@ metadata: release.deckhouse.io/notified: "false" creationTimestamp: null name: v1.26.0 - resourceVersion: "1003" + resourceVersion: "1002" spec: version: v1.26.0 status: diff --git a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/auto-deploy-patch-release-in-manual-mode.yaml b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/auto-deploy-patch-release-in-manual-mode.yaml index 64c0e3f22f..22d6b1de60 100644 --- a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/auto-deploy-patch-release-in-manual-mode.yaml +++ b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/auto-deploy-patch-release-in-manual-mode.yaml @@ -18,9 +18,12 @@ apiVersion: deckhouse.io/v1alpha1 approved: false kind: DeckhouseRelease metadata: + annotations: + release.deckhouse.io/isUpdating: "false" + release.deckhouse.io/notified: "true" creationTimestamp: null name: v1.25.1 - resourceVersion: "1001" + resourceVersion: "1002" spec: version: v1.25.1 status: @@ -35,12 +38,12 @@ kind: DeckhouseRelease metadata: creationTimestamp: null name: v1.26.0 - resourceVersion: "1000" + resourceVersion: "1001" spec: version: v1.26.0 status: approved: false - message: Awaiting for v1.25.1 release to be deployed + message: awaiting for v1.25.1 release to be deployed phase: Pending transitionTime: null --- diff --git a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/auto-patch-mode-minor-release.yaml b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/auto-patch-mode-minor-release.yaml index 562e09c791..75056a91a0 100644 --- a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/auto-patch-mode-minor-release.yaml +++ b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/auto-patch-mode-minor-release.yaml @@ -18,9 +18,12 @@ apiVersion: deckhouse.io/v1alpha1 approved: false kind: DeckhouseRelease metadata: + annotations: + release.deckhouse.io/isUpdating: "false" + release.deckhouse.io/notified: "true" creationTimestamp: null name: v1.27.0 - resourceVersion: "1000" + resourceVersion: "1001" spec: version: v1.27.0 status: diff --git a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/auto-patch-mode.yaml b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/auto-patch-mode.yaml index 4bda34a85d..919d97e911 100644 --- a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/auto-patch-mode.yaml +++ b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/auto-patch-mode.yaml @@ -3,9 +3,12 @@ apiVersion: deckhouse.io/v1alpha1 approved: false kind: DeckhouseRelease metadata: + annotations: + release.deckhouse.io/isUpdating: "false" + release.deckhouse.io/notified: "true" creationTimestamp: null name: v1.26.2 - resourceVersion: "1000" + resourceVersion: "1001" spec: version: v1.26.2 status: diff --git a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/auto-patch-patch-update.yaml b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/auto-patch-patch-update.yaml index 2d7aebe655..f50bdc5703 100644 --- a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/auto-patch-patch-update.yaml +++ b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/auto-patch-patch-update.yaml @@ -20,9 +20,11 @@ kind: DeckhouseRelease metadata: annotations: release.deckhouse.io/approved: "true" + release.deckhouse.io/isUpdating: "false" + release.deckhouse.io/notified: "true" creationTimestamp: null name: v1.26.3 - resourceVersion: "1000" + resourceVersion: "1001" spec: version: v1.26.3 status: diff --git a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/deckhouse-previous-release-is-not-ready.yaml b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/deckhouse-previous-release-is-not-ready.yaml index 3403eb6134..5e9a3279d1 100644 --- a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/deckhouse-previous-release-is-not-ready.yaml +++ b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/deckhouse-previous-release-is-not-ready.yaml @@ -18,14 +18,17 @@ apiVersion: deckhouse.io/v1alpha1 approved: false kind: DeckhouseRelease metadata: + annotations: + release.deckhouse.io/isUpdating: "false" + release.deckhouse.io/notified: "true" creationTimestamp: null name: v1.26.0 - resourceVersion: "1000" + resourceVersion: "1001" spec: version: v1.26.0 status: approved: false - message: Awaiting for Deckhouse pod to be ready + message: awaiting for Deckhouse v1.25.0 pod to be ready phase: Pending transitionTime: null --- diff --git a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/disruption-release.yaml b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/disruption-release.yaml index 09afa72cd3..8e1b4e4be2 100644 --- a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/disruption-release.yaml +++ b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/disruption-release.yaml @@ -3,16 +3,19 @@ apiVersion: deckhouse.io/v1alpha1 approved: false kind: DeckhouseRelease metadata: + annotations: + release.deckhouse.io/isUpdating: "false" + release.deckhouse.io/notified: "false" creationTimestamp: null name: v1.36.0 - resourceVersion: "1000" + resourceVersion: "1001" spec: disruptions: - testme version: v1.36.0 status: approved: false - message: 'Release requires disruption approval (`kubectl annotate DeckhouseRelease + message: 'release blocked, disruption approval required: (`kubectl annotate DeckhouseRelease v1.36.0 release.deckhouse.io/disruption-approved=true`): some test reason' phase: Pending transitionTime: null diff --git a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/existed-release-suspended.yaml b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/existed-release-suspended.yaml index ba7d8d4d03..2aa8366ed7 100644 --- a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/existed-release-suspended.yaml +++ b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/existed-release-suspended.yaml @@ -1,5 +1,25 @@ --- apiVersion: deckhouse.io/v1alpha1 +approved: true +kind: DeckhouseRelease +metadata: + annotations: + release.deckhouse.io/current-restored: "true" + release.deckhouse.io/isUpdating: "false" + release.deckhouse.io/notified: "false" + creationTimestamp: null + labels: + heritage: deckhouse + name: v1.15.0 + resourceVersion: "1" +spec: + version: v1.15.0 +status: + approved: false + message: "" + transitionTime: null +--- +apiVersion: deckhouse.io/v1alpha1 approved: false kind: DeckhouseRelease metadata: diff --git a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/few-minor-releases-version-more-than-one.yaml b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/few-minor-releases-version-more-than-one.yaml new file mode 100644 index 0000000000..01b9bccdf1 --- /dev/null +++ b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/few-minor-releases-version-more-than-one.yaml @@ -0,0 +1,105 @@ +--- +apiVersion: deckhouse.io/v1alpha1 +approved: false +kind: DeckhouseRelease +metadata: + creationTimestamp: null + name: v1.31.0 + resourceVersion: "999" +spec: + version: v1.31.0 +status: + approved: false + message: "" + phase: Deployed + transitionTime: null +--- +apiVersion: deckhouse.io/v1alpha1 +approved: false +kind: DeckhouseRelease +metadata: + creationTimestamp: null + name: v1.33.0 + resourceVersion: "1000" +spec: + version: v1.33.0 +status: + approved: false + message: minor version is more than deployed v1.31.0 by one + phase: Pending + transitionTime: null +--- +apiVersion: deckhouse.io/v1alpha1 +approved: false +kind: DeckhouseRelease +metadata: + creationTimestamp: null + name: v1.34.0 + resourceVersion: "1000" +spec: + version: v1.34.0 +status: + approved: false + message: awaiting for v1.33.0 release to be deployed + phase: Pending + transitionTime: null +--- +apiVersion: deckhouse.io/v1alpha1 +approved: false +kind: DeckhouseRelease +metadata: + creationTimestamp: null + name: v1.35.0 + resourceVersion: "1000" +spec: + version: v1.35.0 +status: + approved: false + message: awaiting for v1.33.0 release to be deployed + phase: Pending + transitionTime: null +--- +apiVersion: v1 +kind: Pod +metadata: + creationTimestamp: null + labels: + app: deckhouse + name: deckhouse-6f46df5bd7-nk4j7 + namespace: d8-system + resourceVersion: "999" +spec: + containers: + - image: dev-registry.deckhouse.io/sys/deckhouse-oss:v1.2.3 + name: deckhouse + resources: {} +status: + containerStatuses: + - containerID: containerd://9990d3eccb8657d0bfe755672308831b6d0fab7f3aac553487c60bf0f076b2e3 + image: "" + imageID: dev-registry.deckhouse.io/sys/deckhouse-oss/dev@sha256:d57f01a88e54f863ff5365c989cb4e2654398fa274d46389e0af749090b862d1 + lastState: {} + name: "" + ready: true + restartCount: 0 + state: {} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + creationTimestamp: null + name: deckhouse + namespace: d8-system + resourceVersion: "999" +spec: + selector: null + strategy: {} + template: + metadata: + creationTimestamp: null + spec: + containers: + - image: my.registry.com/deckhouse:v1.25.0 + name: deckhouse + resources: {} +status: {} diff --git a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/few-minor-releases.yaml b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/few-minor-releases.yaml new file mode 100644 index 0000000000..6932bdf0ca --- /dev/null +++ b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/few-minor-releases.yaml @@ -0,0 +1,129 @@ +--- +apiVersion: deckhouse.io/v1alpha1 +approved: false +kind: DeckhouseRelease +metadata: + annotations: + release.deckhouse.io/isUpdating: "false" + release.deckhouse.io/notified: "true" + creationTimestamp: null + name: v1.31.0 + resourceVersion: "1001" +spec: + version: v1.31.0 +status: + approved: false + message: "" + phase: Superseded + transitionTime: "2019-10-17T15:33:00Z" +--- +apiVersion: deckhouse.io/v1alpha1 +approved: false +kind: DeckhouseRelease +metadata: + annotations: + release.deckhouse.io/isUpdating: "true" + release.deckhouse.io/notified: "false" + creationTimestamp: null + name: v1.32.0 + resourceVersion: "1001" +spec: + version: v1.32.0 +status: + approved: false + message: "" + phase: Deployed + transitionTime: "2019-10-17T15:33:00Z" +--- +apiVersion: deckhouse.io/v1alpha1 +approved: false +kind: DeckhouseRelease +metadata: + annotations: + release.deckhouse.io/isUpdating: "false" + release.deckhouse.io/notified: "true" + creationTimestamp: null + name: v1.33.0 + resourceVersion: "1001" +spec: + version: v1.33.0 +status: + approved: false + message: awaiting for Deckhouse v1.32.0 pod to be ready + phase: Pending + transitionTime: null +--- +apiVersion: deckhouse.io/v1alpha1 +approved: false +kind: DeckhouseRelease +metadata: + creationTimestamp: null + name: v1.34.0 + resourceVersion: "1000" +spec: + version: v1.34.0 +status: + approved: false + message: awaiting for Deckhouse v1.32.0 pod to be ready + phase: Pending + transitionTime: null +--- +apiVersion: deckhouse.io/v1alpha1 +approved: false +kind: DeckhouseRelease +metadata: + creationTimestamp: null + name: v1.35.0 + resourceVersion: "1000" +spec: + version: v1.35.0 +status: + approved: false + message: awaiting for Deckhouse v1.32.0 pod to be ready + phase: Pending + transitionTime: null +--- +apiVersion: v1 +kind: Pod +metadata: + creationTimestamp: null + labels: + app: deckhouse + name: deckhouse-6f46df5bd7-nk4j7 + namespace: d8-system + resourceVersion: "999" +spec: + containers: + - image: dev-registry.deckhouse.io/sys/deckhouse-oss:v1.2.3 + name: deckhouse + resources: {} +status: + containerStatuses: + - containerID: containerd://9990d3eccb8657d0bfe755672308831b6d0fab7f3aac553487c60bf0f076b2e3 + image: "" + imageID: dev-registry.deckhouse.io/sys/deckhouse-oss/dev@sha256:d57f01a88e54f863ff5365c989cb4e2654398fa274d46389e0af749090b862d1 + lastState: {} + name: "" + ready: true + restartCount: 0 + state: {} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + creationTimestamp: null + name: deckhouse + namespace: d8-system + resourceVersion: "1000" +spec: + selector: null + strategy: {} + template: + metadata: + creationTimestamp: null + spec: + containers: + - image: my.registry.com/deckhouse:v1.32.0 + name: deckhouse + resources: {} +status: {} diff --git a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/few-patch-releases.yaml b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/few-patch-releases.yaml index c97a04d011..2875ff04f6 100644 --- a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/few-patch-releases.yaml +++ b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/few-patch-releases.yaml @@ -5,14 +5,14 @@ kind: DeckhouseRelease metadata: creationTimestamp: null name: v1.31.0 - resourceVersion: "999" + resourceVersion: "1000" spec: version: v1.31.0 status: approved: false message: "" - phase: Deployed - transitionTime: null + phase: Superseded + transitionTime: "2019-10-17T15:33:00Z" --- apiVersion: deckhouse.io/v1alpha1 approved: false @@ -48,29 +48,35 @@ apiVersion: deckhouse.io/v1alpha1 approved: false kind: DeckhouseRelease metadata: + annotations: + release.deckhouse.io/isUpdating: "true" + release.deckhouse.io/notified: "false" creationTimestamp: null name: v1.31.3 - resourceVersion: "999" + resourceVersion: "1001" spec: version: v1.31.3 status: approved: false message: "" - phase: Pending - transitionTime: null + phase: Deployed + transitionTime: "2019-10-17T15:33:00Z" --- apiVersion: deckhouse.io/v1alpha1 approved: false kind: DeckhouseRelease metadata: + annotations: + release.deckhouse.io/isUpdating: "false" + release.deckhouse.io/notified: "true" creationTimestamp: null name: v1.32.0 - resourceVersion: "1000" + resourceVersion: "1001" spec: version: v1.32.0 status: approved: false - message: Awaiting for v1.31.3 release to be deployed + message: awaiting for Deckhouse v1.31.3 pod to be ready phase: Pending transitionTime: null --- @@ -105,7 +111,7 @@ metadata: creationTimestamp: null name: deckhouse namespace: d8-system - resourceVersion: "999" + resourceVersion: "1000" spec: selector: null strategy: {} @@ -114,7 +120,7 @@ spec: creationTimestamp: null spec: containers: - - image: my.registry.com/deckhouse:v1.25.0 + - image: my.registry.com/deckhouse:v1.31.3 name: deckhouse resources: {} status: {} diff --git a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/forced-few-minor-releases.yaml b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/forced-few-minor-releases.yaml new file mode 100644 index 0000000000..60e6bc7104 --- /dev/null +++ b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/forced-few-minor-releases.yaml @@ -0,0 +1,123 @@ +--- +apiVersion: deckhouse.io/v1alpha1 +approved: false +kind: DeckhouseRelease +metadata: + creationTimestamp: null + name: v1.31.0 + resourceVersion: "1000" +spec: + version: v1.31.0 +status: + approved: false + message: "" + phase: Superseded + transitionTime: "2019-10-17T15:33:00Z" +--- +apiVersion: deckhouse.io/v1alpha1 +approved: false +kind: DeckhouseRelease +metadata: + creationTimestamp: null + name: v1.32.0 + resourceVersion: "1000" +spec: + version: v1.32.0 +status: + approved: false + message: "" + phase: Skipped + transitionTime: "2019-10-17T15:33:00Z" +--- +apiVersion: deckhouse.io/v1alpha1 +approved: false +kind: DeckhouseRelease +metadata: + creationTimestamp: null + name: v1.33.0 + resourceVersion: "1000" +spec: + version: v1.33.0 +status: + approved: false + message: "" + phase: Skipped + transitionTime: "2019-10-17T15:33:00Z" +--- +apiVersion: deckhouse.io/v1alpha1 +approved: false +kind: DeckhouseRelease +metadata: + creationTimestamp: null + name: v1.34.0 + resourceVersion: "1000" +spec: + version: v1.34.0 +status: + approved: false + message: "" + phase: Skipped + transitionTime: "2019-10-17T15:33:00Z" +--- +apiVersion: deckhouse.io/v1alpha1 +approved: false +kind: DeckhouseRelease +metadata: + annotations: + release.deckhouse.io/isUpdating: "true" + release.deckhouse.io/notified: "false" + creationTimestamp: null + name: v1.35.0 + resourceVersion: "1001" +spec: + version: v1.35.0 +status: + approved: false + message: "" + phase: Deployed + transitionTime: "2019-10-17T15:33:00Z" +--- +apiVersion: v1 +kind: Pod +metadata: + creationTimestamp: null + labels: + app: deckhouse + name: deckhouse-6f46df5bd7-nk4j7 + namespace: d8-system + resourceVersion: "999" +spec: + containers: + - image: dev-registry.deckhouse.io/sys/deckhouse-oss:v1.2.3 + name: deckhouse + resources: {} +status: + containerStatuses: + - containerID: containerd://9990d3eccb8657d0bfe755672308831b6d0fab7f3aac553487c60bf0f076b2e3 + image: "" + imageID: dev-registry.deckhouse.io/sys/deckhouse-oss/dev@sha256:d57f01a88e54f863ff5365c989cb4e2654398fa274d46389e0af749090b862d1 + lastState: {} + name: "" + ready: true + restartCount: 0 + state: {} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + creationTimestamp: null + name: deckhouse + namespace: d8-system + resourceVersion: "1000" +spec: + selector: null + strategy: {} + template: + metadata: + creationTimestamp: null + spec: + containers: + - image: my.registry.com/deckhouse:v1.35.0 + name: deckhouse + resources: {} +status: {} diff --git a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/forced-release.yaml b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/forced-release.yaml index 13f1c64806..e901272dcf 100644 --- a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/forced-release.yaml +++ b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/forced-release.yaml @@ -5,13 +5,13 @@ kind: DeckhouseRelease metadata: creationTimestamp: null name: v1.31.0 - resourceVersion: "1001" + resourceVersion: "1000" spec: version: v1.31.0 status: approved: false message: "" - phase: Superseded + phase: Skipped transitionTime: "2019-10-17T15:33:00Z" --- apiVersion: deckhouse.io/v1alpha1 @@ -23,7 +23,7 @@ metadata: release.deckhouse.io/notified: "false" creationTimestamp: null name: v1.31.1 - resourceVersion: "1002" + resourceVersion: "1001" spec: version: v1.31.1 status: diff --git a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/have-canary-release-wave-0.yaml b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/have-canary-release-wave-0.yaml index b67b3ea4ad..855c12424d 100644 --- a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/have-canary-release-wave-0.yaml +++ b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/have-canary-release-wave-0.yaml @@ -1,17 +1,19 @@ --- apiVersion: deckhouse.io/v1alpha1 -approved: false +approved: true kind: DeckhouseRelease metadata: annotations: + release.deckhouse.io/current-restored: "true" release.deckhouse.io/isUpdating: "false" release.deckhouse.io/notified: "false" creationTimestamp: null - name: v1.25.0 + labels: + heritage: deckhouse + name: v1.15.0 resourceVersion: "1" spec: - changelogLink: https://github.com/deckhouse/deckhouse/releases/tag/v1.25.0 - version: v1.25.0 + version: v1.15.0 status: approved: false message: "" diff --git a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/have-canary-release-wave-4.yaml b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/have-canary-release-wave-4.yaml index b5daffa6d2..855c12424d 100644 --- a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/have-canary-release-wave-4.yaml +++ b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/have-canary-release-wave-4.yaml @@ -1,18 +1,19 @@ --- apiVersion: deckhouse.io/v1alpha1 -approved: false +approved: true kind: DeckhouseRelease metadata: annotations: + release.deckhouse.io/current-restored: "true" release.deckhouse.io/isUpdating: "false" release.deckhouse.io/notified: "false" creationTimestamp: null - name: v1.25.5 + labels: + heritage: deckhouse + name: v1.15.0 resourceVersion: "1" spec: - applyAfter: "2019-10-17T16:33:00Z" - changelogLink: https://github.com/deckhouse/deckhouse/releases/tag/v1.25.5 - version: v1.25.5 + version: v1.15.0 status: approved: false message: "" diff --git a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/have-new-deckhouse-image.yaml b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/have-new-deckhouse-image.yaml index 3b13642418..855c12424d 100644 --- a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/have-new-deckhouse-image.yaml +++ b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/have-new-deckhouse-image.yaml @@ -1,17 +1,19 @@ --- apiVersion: deckhouse.io/v1alpha1 -approved: false +approved: true kind: DeckhouseRelease metadata: annotations: + release.deckhouse.io/current-restored: "true" release.deckhouse.io/isUpdating: "false" release.deckhouse.io/notified: "false" creationTimestamp: null - name: v1.25.3 + labels: + heritage: deckhouse + name: v1.15.0 resourceVersion: "1" spec: - changelogLink: https://github.com/deckhouse/deckhouse/releases/tag/v1.25.3 - version: v1.25.3 + version: v1.15.0 status: approved: false message: "" diff --git a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/image-hash-not-changed.yaml b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/image-hash-not-changed.yaml index e69de29bb2..855c12424d 100644 --- a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/image-hash-not-changed.yaml +++ b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/image-hash-not-changed.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: deckhouse.io/v1alpha1 +approved: true +kind: DeckhouseRelease +metadata: + annotations: + release.deckhouse.io/current-restored: "true" + release.deckhouse.io/isUpdating: "false" + release.deckhouse.io/notified: "false" + creationTimestamp: null + labels: + heritage: deckhouse + name: v1.15.0 + resourceVersion: "1" +spec: + version: v1.15.0 +status: + approved: false + message: "" + transitionTime: null diff --git a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/inherit-release-cooldown.yaml b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/inherit-release-cooldown.yaml index 793b8d4364..855c12424d 100644 --- a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/inherit-release-cooldown.yaml +++ b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/inherit-release-cooldown.yaml @@ -1,18 +1,19 @@ --- apiVersion: deckhouse.io/v1alpha1 -approved: false +approved: true kind: DeckhouseRelease metadata: annotations: - release.deckhouse.io/cooldown: "2026-06-06T16:16:16Z" + release.deckhouse.io/current-restored: "true" release.deckhouse.io/isUpdating: "false" release.deckhouse.io/notified: "false" creationTimestamp: null - name: v1.31.1 + labels: + heritage: deckhouse + name: v1.15.0 resourceVersion: "1" spec: - changelogLink: https://github.com/deckhouse/deckhouse/releases/tag/v1.31.1 - version: v1.31.1 + version: v1.15.0 status: approved: false message: "" diff --git a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/manual-approval-mode-is-set.yaml b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/manual-approval-mode-is-set.yaml index 2a75e9bf9a..bad8528fe5 100644 --- a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/manual-approval-mode-is-set.yaml +++ b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/manual-approval-mode-is-set.yaml @@ -18,9 +18,12 @@ apiVersion: deckhouse.io/v1alpha1 approved: false kind: DeckhouseRelease metadata: + annotations: + release.deckhouse.io/isUpdating: "false" + release.deckhouse.io/notified: "true" creationTimestamp: null name: v1.26.0 - resourceVersion: "1001" + resourceVersion: "1002" spec: version: v1.26.0 status: diff --git a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/manual-approval-mode-with-canary-process.yaml b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/manual-approval-mode-with-canary-process.yaml index 5249520372..7b8d60d80e 100644 --- a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/manual-approval-mode-with-canary-process.yaml +++ b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/manual-approval-mode-with-canary-process.yaml @@ -18,9 +18,12 @@ apiVersion: deckhouse.io/v1alpha1 approved: false kind: DeckhouseRelease metadata: + annotations: + release.deckhouse.io/isUpdating: "false" + release.deckhouse.io/notified: "true" creationTimestamp: null name: v1.36.0 - resourceVersion: "1001" + resourceVersion: "1002" spec: applyAfter: "2222-11-11T23:23:23Z" version: v1.36.0 diff --git a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/manual-mode.yaml b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/manual-mode.yaml index 7928d0e26b..05566b94b3 100644 --- a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/manual-mode.yaml +++ b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/manual-mode.yaml @@ -18,9 +18,12 @@ apiVersion: deckhouse.io/v1alpha1 approved: false kind: DeckhouseRelease metadata: + annotations: + release.deckhouse.io/isUpdating: "false" + release.deckhouse.io/notified: "true" creationTimestamp: null name: v1.27.0 - resourceVersion: "1001" + resourceVersion: "1002" spec: version: v1.27.0 status: diff --git a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/new-release-suspended.yaml b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/new-release-suspended.yaml index afbc485bcd..855c12424d 100644 --- a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/new-release-suspended.yaml +++ b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/new-release-suspended.yaml @@ -1,18 +1,19 @@ --- apiVersion: deckhouse.io/v1alpha1 -approved: false +approved: true kind: DeckhouseRelease metadata: annotations: + release.deckhouse.io/current-restored: "true" release.deckhouse.io/isUpdating: "false" release.deckhouse.io/notified: "false" - release.deckhouse.io/suspended: "true" creationTimestamp: null - name: v1.25.0 + labels: + heritage: deckhouse + name: v1.15.0 resourceVersion: "1" spec: - changelogLink: https://github.com/deckhouse/deckhouse/releases/tag/v1.25.0 - version: v1.25.0 + version: v1.15.0 status: approved: false message: "" diff --git a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/notification-basic-auth.yaml b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/notification-basic-auth.yaml index fbaec54696..ee1bfc1665 100644 --- a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/notification-basic-auth.yaml +++ b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/notification-basic-auth.yaml @@ -29,7 +29,7 @@ spec: version: v1.36.0 status: approved: false - message: Release is postponed by canary process until 11 Nov 22 23:23 UTC + message: Release is postponed until 11 Nov 22 23:23 UTC phase: Pending transitionTime: null --- diff --git a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/notification-bearer-token-auth.yaml b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/notification-bearer-token-auth.yaml index fbaec54696..ee1bfc1665 100644 --- a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/notification-bearer-token-auth.yaml +++ b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/notification-bearer-token-auth.yaml @@ -29,7 +29,7 @@ spec: version: v1.36.0 status: approved: false - message: Release is postponed by canary process until 11 Nov 22 23:23 UTC + message: Release is postponed until 11 Nov 22 23:23 UTC phase: Pending transitionTime: null --- diff --git a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/notification-release-apply-after-time-is-after-notification-period.yaml b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/notification-release-apply-after-time-is-after-notification-period.yaml index fbaec54696..7102627717 100644 --- a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/notification-release-apply-after-time-is-after-notification-period.yaml +++ b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/notification-release-apply-after-time-is-after-notification-period.yaml @@ -20,6 +20,7 @@ kind: DeckhouseRelease metadata: annotations: release.deckhouse.io/isUpdating: "false" + release.deckhouse.io/notification-time-shift: "true" release.deckhouse.io/notified: "true" creationTimestamp: null name: v1.36.0 @@ -29,7 +30,7 @@ spec: version: v1.36.0 status: approved: false - message: Release is postponed by canary process until 11 Nov 22 23:23 UTC + message: Release is postponed by notification until 11 Nov 22 23:23 UTC phase: Pending transitionTime: null --- diff --git a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/patch-out-of-update-window.yaml b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/patch-out-of-update-window.yaml index c6dc49bfb2..9c6a0c1578 100644 --- a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/patch-out-of-update-window.yaml +++ b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/patch-out-of-update-window.yaml @@ -43,7 +43,7 @@ spec: version: v1.26.0 status: approved: false - message: Awaiting for v1.25.1 release to be deployed + message: awaiting for v1.25.1 release to be deployed phase: Pending transitionTime: null --- diff --git a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/patch-release-has-own-cooldown.yaml b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/patch-release-has-own-cooldown.yaml index aa93ae9981..855c12424d 100644 --- a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/patch-release-has-own-cooldown.yaml +++ b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/patch-release-has-own-cooldown.yaml @@ -1,18 +1,19 @@ --- apiVersion: deckhouse.io/v1alpha1 -approved: false +approved: true kind: DeckhouseRelease metadata: annotations: - release.deckhouse.io/cooldown: "2030-05-05T15:15:15Z" + release.deckhouse.io/current-restored: "true" release.deckhouse.io/isUpdating: "false" release.deckhouse.io/notified: "false" creationTimestamp: null - name: v1.31.2 + labels: + heritage: deckhouse + name: v1.15.0 resourceVersion: "1" spec: - changelogLink: https://github.com/deckhouse/deckhouse/releases/tag/v1.31.2 - version: v1.31.2 + version: v1.15.0 status: approved: false message: "" diff --git a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/patch-release-notification.yaml b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/patch-release-notification.yaml index e964cdfa7b..13fb03da7e 100644 --- a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/patch-release-notification.yaml +++ b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/patch-release-notification.yaml @@ -24,7 +24,7 @@ metadata: release.deckhouse.io/notified: "true" creationTimestamp: null name: v1.26.0 - resourceVersion: "1002" + resourceVersion: "1001" spec: applyAfter: "2019-10-17T17:33:00Z" version: v1.25.1 diff --git a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/patch-release-with-apply-now-annotation-out-of-window.yaml b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/patch-release-with-apply-now-annotation-out-of-window.yaml index a577f5d86f..81fd67dea5 100644 --- a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/patch-release-with-apply-now-annotation-out-of-window.yaml +++ b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/patch-release-with-apply-now-annotation-out-of-window.yaml @@ -23,7 +23,7 @@ metadata: release.deckhouse.io/notified: "false" creationTimestamp: null name: v1.25.1 - resourceVersion: "1002" + resourceVersion: "1001" spec: version: v1.25.1 status: diff --git a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/pending-manual-release-on-cluster-bootstrap.yaml b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/pending-manual-release-on-cluster-bootstrap.yaml index e255273771..aed58d3eeb 100644 --- a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/pending-manual-release-on-cluster-bootstrap.yaml +++ b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/pending-manual-release-on-cluster-bootstrap.yaml @@ -18,9 +18,12 @@ apiVersion: deckhouse.io/v1alpha1 approved: false kind: DeckhouseRelease metadata: + annotations: + release.deckhouse.io/isUpdating: "false" + release.deckhouse.io/notified: "true" creationTimestamp: null name: v1.46.0 - resourceVersion: "1001" + resourceVersion: "1002" spec: version: v1.46.0 status: diff --git a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/postponed-release.yaml b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/postponed-release.yaml index 57b8527bf5..62b50b9fb5 100644 --- a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/postponed-release.yaml +++ b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/postponed-release.yaml @@ -18,15 +18,18 @@ apiVersion: deckhouse.io/v1alpha1 approved: false kind: DeckhouseRelease metadata: + annotations: + release.deckhouse.io/isUpdating: "false" + release.deckhouse.io/notified: "true" creationTimestamp: null name: v1.25.1 - resourceVersion: "1000" + resourceVersion: "1001" spec: applyAfter: "2222-11-11T23:23:23Z" version: v1.25.1 status: approved: false - message: Release is postponed by canary process until 11 Nov 22 23:23 UTC + message: Release is postponed until 11 Nov 22 23:23 UTC phase: Pending transitionTime: null --- diff --git a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/release-has-canary.yaml b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/release-has-canary.yaml index ba4dcfe226..855c12424d 100644 --- a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/release-has-canary.yaml +++ b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/release-has-canary.yaml @@ -1,18 +1,19 @@ --- apiVersion: deckhouse.io/v1alpha1 -approved: false +approved: true kind: DeckhouseRelease metadata: annotations: + release.deckhouse.io/current-restored: "true" release.deckhouse.io/isUpdating: "false" release.deckhouse.io/notified: "false" creationTimestamp: null - name: v1.31.0 + labels: + heritage: deckhouse + name: v1.15.0 resourceVersion: "1" spec: - applyAfter: "2019-10-17T17:03:00Z" - changelogLink: https://github.com/deckhouse/deckhouse/releases/tag/v1.31.0 - version: v1.31.0 + version: v1.15.0 status: approved: false message: "" diff --git a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/release-has-cooldown.yaml b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/release-has-cooldown.yaml index faf76bf86c..855c12424d 100644 --- a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/release-has-cooldown.yaml +++ b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/release-has-cooldown.yaml @@ -1,18 +1,19 @@ --- apiVersion: deckhouse.io/v1alpha1 -approved: false +approved: true kind: DeckhouseRelease metadata: annotations: - release.deckhouse.io/cooldown: "2026-06-06T16:16:16Z" + release.deckhouse.io/current-restored: "true" release.deckhouse.io/isUpdating: "false" release.deckhouse.io/notified: "false" creationTimestamp: null - name: v1.31.0 + labels: + heritage: deckhouse + name: v1.15.0 resourceVersion: "1" spec: - changelogLink: https://github.com/deckhouse/deckhouse/releases/tag/v1.31.0 - version: v1.31.0 + version: v1.15.0 status: approved: false message: "" diff --git a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/release-has-disruptions.yaml b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/release-has-disruptions.yaml index 408bbe1c23..855c12424d 100644 --- a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/release-has-disruptions.yaml +++ b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/release-has-disruptions.yaml @@ -1,19 +1,19 @@ --- apiVersion: deckhouse.io/v1alpha1 -approved: false +approved: true kind: DeckhouseRelease metadata: annotations: + release.deckhouse.io/current-restored: "true" release.deckhouse.io/isUpdating: "false" release.deckhouse.io/notified: "false" creationTimestamp: null - name: v1.32.0 + labels: + heritage: deckhouse + name: v1.15.0 resourceVersion: "1" spec: - changelogLink: https://github.com/deckhouse/deckhouse/releases/tag/v1.32.0 - disruptions: - - ingressNginx - version: v1.32.0 + version: v1.15.0 status: approved: false message: "" diff --git a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/release-has-requirements.yaml b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/release-has-requirements.yaml index 5d93964b18..855c12424d 100644 --- a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/release-has-requirements.yaml +++ b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/release-has-requirements.yaml @@ -1,20 +1,19 @@ --- apiVersion: deckhouse.io/v1alpha1 -approved: false +approved: true kind: DeckhouseRelease metadata: annotations: + release.deckhouse.io/current-restored: "true" release.deckhouse.io/isUpdating: "false" release.deckhouse.io/notified: "false" creationTimestamp: null - name: v1.30.0 + labels: + heritage: deckhouse + name: v1.15.0 resourceVersion: "1" spec: - changelogLink: https://github.com/deckhouse/deckhouse/releases/tag/v1.30.0 - requirements: - k8s: "1.19" - req1: dep1 - version: v1.30.0 + version: v1.15.0 status: approved: false message: "" diff --git a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/release-with-apply-now-annotation-disruption.yaml b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/release-with-apply-now-annotation-disruption.yaml index cfa919923f..6beb83988b 100644 --- a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/release-with-apply-now-annotation-disruption.yaml +++ b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/release-with-apply-now-annotation-disruption.yaml @@ -23,7 +23,7 @@ metadata: release.deckhouse.io/notified: "false" creationTimestamp: null name: v1.26.0 - resourceVersion: "1002" + resourceVersion: "1001" spec: disruptions: - testme diff --git a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/release-with-apply-now-annotation-out-of-window.yaml b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/release-with-apply-now-annotation-out-of-window.yaml index 4a7a6de50c..3ce21b3feb 100644 --- a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/release-with-apply-now-annotation-out-of-window.yaml +++ b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/release-with-apply-now-annotation-out-of-window.yaml @@ -23,7 +23,7 @@ metadata: release.deckhouse.io/notified: "false" creationTimestamp: null name: v1.26.0 - resourceVersion: "1002" + resourceVersion: "1001" spec: version: v1.26.0 status: diff --git a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/release-with-changelog.yaml b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/release-with-changelog.yaml index 22751927b9..855c12424d 100644 --- a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/release-with-changelog.yaml +++ b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/release-with-changelog.yaml @@ -1,37 +1,19 @@ --- apiVersion: deckhouse.io/v1alpha1 -approved: false +approved: true kind: DeckhouseRelease metadata: annotations: + release.deckhouse.io/current-restored: "true" release.deckhouse.io/isUpdating: "false" release.deckhouse.io/notified: "false" creationTimestamp: null - name: v1.31.0 + labels: + heritage: deckhouse + name: v1.15.0 resourceVersion: "1" spec: - changelog: - cert-manager: - fixes: - - pull_request: https://github.com/deckhouse/deckhouse/pull/999 - summary: Remove D8CertmanagerOrphanSecretsWithoutCorrespondingCertificateResources - global: - features: - - description: All master nodes will have `control-plane` role in new exist - clusters. - note: Add migration for adding role. Bashible steps will be rerunned on master - nodes. - pull_request: https://github.com/deckhouse/deckhouse/pull/562 - - description: Update Kubernetes patch versions. - pull_request: https://github.com/deckhouse/deckhouse/pull/558 - fixes: - - description: Fix parsing deckhouse images repo if there is the sha256 sum - in the image name - pull_request: https://github.com/deckhouse/deckhouse/pull/527 - - description: Fix serialization of empty strings in secrets - pull_request: https://github.com/deckhouse/deckhouse/pull/523 - changelogLink: https://github.com/deckhouse/deckhouse/releases/tag/v1.31.0 - version: v1.31.0 + version: v1.15.0 status: approved: false message: "" diff --git a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/release-with-notification-settings.yaml b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/release-with-notification-settings.yaml index efdfa3c149..c420518f35 100644 --- a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/release-with-notification-settings.yaml +++ b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/release-with-notification-settings.yaml @@ -24,7 +24,7 @@ metadata: release.deckhouse.io/notified: "true" creationTimestamp: null name: v1.26.0 - resourceVersion: "1002" + resourceVersion: "1001" spec: applyAfter: "2019-10-17T16:33:00Z" version: v1.26.0 diff --git a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/restore-absent-releases-from-registry.yaml b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/restore-absent-releases-from-registry.yaml index f5da24331d..5ab1268999 100644 --- a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/restore-absent-releases-from-registry.yaml +++ b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/restore-absent-releases-from-registry.yaml @@ -22,6 +22,8 @@ metadata: release.deckhouse.io/isUpdating: "false" release.deckhouse.io/notified: "false" creationTimestamp: null + labels: + heritage: deckhouse name: v1.58.1 resourceVersion: "1" spec: @@ -40,6 +42,8 @@ metadata: release.deckhouse.io/isUpdating: "false" release.deckhouse.io/notified: "false" creationTimestamp: null + labels: + heritage: deckhouse name: v1.59.3 resourceVersion: "1" spec: @@ -58,6 +62,8 @@ metadata: release.deckhouse.io/isUpdating: "false" release.deckhouse.io/notified: "false" creationTimestamp: null + labels: + heritage: deckhouse name: v1.60.2 resourceVersion: "1" spec: diff --git a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/resume-suspended-release.yaml b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/resume-suspended-release.yaml index 6fbae6f5ad..d886c50e8f 100644 --- a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/resume-suspended-release.yaml +++ b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/resume-suspended-release.yaml @@ -1,5 +1,25 @@ --- apiVersion: deckhouse.io/v1alpha1 +approved: true +kind: DeckhouseRelease +metadata: + annotations: + release.deckhouse.io/current-restored: "true" + release.deckhouse.io/isUpdating: "false" + release.deckhouse.io/notified: "false" + creationTimestamp: null + labels: + heritage: deckhouse + name: v1.15.0 + resourceVersion: "1" +spec: + version: v1.15.0 +status: + approved: false + message: "" + transitionTime: null +--- +apiVersion: deckhouse.io/v1alpha1 approved: false kind: DeckhouseRelease metadata: diff --git a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/second-run-of-the-hook-in-a-manual-mode-should-not-change-state.yaml b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/second-run-of-the-hook-in-a-manual-mode-should-not-change-state.yaml index 63c3db1a06..5d510496a3 100644 --- a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/second-run-of-the-hook-in-a-manual-mode-should-not-change-state.yaml +++ b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/second-run-of-the-hook-in-a-manual-mode-should-not-change-state.yaml @@ -18,9 +18,12 @@ apiVersion: deckhouse.io/v1alpha1 approved: false kind: DeckhouseRelease metadata: + annotations: + release.deckhouse.io/isUpdating: "false" + release.deckhouse.io/notified: "true" creationTimestamp: null name: v1.27.0 - resourceVersion: "1002" + resourceVersion: "1003" spec: version: v1.27.0 status: diff --git a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/step-by-step-update-successfully.yaml b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/step-by-step-update-successfully.yaml index 219efad8df..9270d9edd2 100644 --- a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/step-by-step-update-successfully.yaml +++ b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/step-by-step-update-successfully.yaml @@ -22,6 +22,8 @@ metadata: release.deckhouse.io/isUpdating: "false" release.deckhouse.io/notified: "false" creationTimestamp: null + labels: + heritage: deckhouse name: v1.32.3 resourceVersion: "1" spec: @@ -40,6 +42,8 @@ metadata: release.deckhouse.io/isUpdating: "false" release.deckhouse.io/notified: "false" creationTimestamp: null + labels: + heritage: deckhouse name: v1.33.1 resourceVersion: "1" spec: diff --git a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/suspend-release.yaml b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/suspend-release.yaml index 7b86fa9404..77ce71999e 100644 --- a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/suspend-release.yaml +++ b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/suspend-release.yaml @@ -18,11 +18,9 @@ apiVersion: deckhouse.io/v1alpha1 approved: false kind: DeckhouseRelease metadata: - annotations: - release.deckhouse.io/suspended: "true" creationTimestamp: null name: v1.25.1 - resourceVersion: "1000" + resourceVersion: "1002" spec: version: v1.25.1 status: diff --git a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/unknown-mode-minor-release.yaml b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/unknown-mode-minor-release.yaml index 562e09c791..75056a91a0 100644 --- a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/unknown-mode-minor-release.yaml +++ b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/unknown-mode-minor-release.yaml @@ -18,9 +18,12 @@ apiVersion: deckhouse.io/v1alpha1 approved: false kind: DeckhouseRelease metadata: + annotations: + release.deckhouse.io/isUpdating: "false" + release.deckhouse.io/notified: "true" creationTimestamp: null name: v1.27.0 - resourceVersion: "1000" + resourceVersion: "1001" spec: version: v1.27.0 status: diff --git a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/update-minimal-notification-time-without-configuring-notification-webhook.yaml b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/update-minimal-notification-time-without-configuring-notification-webhook.yaml index 9908367a84..a7dbc7db0f 100644 --- a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/update-minimal-notification-time-without-configuring-notification-webhook.yaml +++ b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/update-minimal-notification-time-without-configuring-notification-webhook.yaml @@ -24,7 +24,7 @@ metadata: release.deckhouse.io/notified: "true" creationTimestamp: null name: v1.26.0 - resourceVersion: "1002" + resourceVersion: "1001" spec: applyAfter: "2019-10-17T17:33:00Z" version: v1.26.0 diff --git a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/update-out-of-day-window.yaml b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/update-out-of-day-window.yaml index d04cbd05eb..415b4a2c63 100644 --- a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/update-out-of-day-window.yaml +++ b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/update-out-of-day-window.yaml @@ -18,9 +18,12 @@ apiVersion: deckhouse.io/v1alpha1 approved: false kind: DeckhouseRelease metadata: + annotations: + release.deckhouse.io/isUpdating: "false" + release.deckhouse.io/notified: "true" creationTimestamp: null name: v1.26.0 - resourceVersion: "1000" + resourceVersion: "1001" spec: version: v1.26.0 status: diff --git a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/update-out-of-window.yaml b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/update-out-of-window.yaml index b7ba4fc5db..b240b50234 100644 --- a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/update-out-of-window.yaml +++ b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/update-out-of-window.yaml @@ -18,9 +18,12 @@ apiVersion: deckhouse.io/v1alpha1 approved: false kind: DeckhouseRelease metadata: + annotations: + release.deckhouse.io/isUpdating: "false" + release.deckhouse.io/notified: "true" creationTimestamp: null name: v1.26.0 - resourceVersion: "1000" + resourceVersion: "1001" spec: version: v1.26.0 status: diff --git a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/update-release-is-deployed.yaml b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/update-release-is-deployed.yaml index 94e5fba1df..ef442652e5 100644 --- a/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/update-release-is-deployed.yaml +++ b/deckhouse-controller/pkg/controller/deckhouse-release/testdata/golden/update-release-is-deployed.yaml @@ -5,7 +5,7 @@ kind: DeckhouseRelease metadata: annotations: release.deckhouse.io/isUpdating: "false" - release.deckhouse.io/notified: "false" + release.deckhouse.io/notified: "true" creationTimestamp: null name: v1.26.0 resourceVersion: "1000" diff --git a/deckhouse-controller/pkg/controller/deckhouse-release/updater/deploy_delay_reason.go b/deckhouse-controller/pkg/controller/deckhouse-release/updater/deploy_delay_reason.go new file mode 100644 index 0000000000..dafb02f5b0 --- /dev/null +++ b/deckhouse-controller/pkg/controller/deckhouse-release/updater/deploy_delay_reason.go @@ -0,0 +1,157 @@ +/* +Copyright 2024 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package d8updater + +import ( + "fmt" + "slices" + "strings" + "time" + + "github.com/deckhouse/deckhouse/deckhouse-controller/pkg/apis/deckhouse.io/v1alpha1" +) + +type deployDelayReason byte + +const ( + noDelay deployDelayReason = 0 + cooldownDelayReason deployDelayReason = 1 << iota + canaryDelayReason + notificationDelayReason + outOfWindowReason + manualApprovalRequiredReason +) + +var deployDelayReasonsStr = map[deployDelayReason]string{ + cooldownDelayReason: "cooldownDelayReason", + canaryDelayReason: "canaryDelayReason", + notificationDelayReason: "notificationDelayReason", + outOfWindowReason: "outOfWindowReason", + manualApprovalRequiredReason: "manualApprovalRequiredReason", +} + +const ( + cooldownDelayMsg = "in cooldown" + canaryDelayReasonMsg = "postponed" + waitingManualApprovalTemplate = "waiting for the '%s: \"true\"' annotation" + outOfWindowMsg = "waiting for the update window" + notificationDelayMsg = "postponed by notification" +) + +func (r deployDelayReason) IsNoDelay() bool { + return r == noDelay +} + +func (r deployDelayReason) String() string { + reasons := r.splitReasons() + if len(reasons) != 0 { + return strings.Join(reasons, ", ") + } + + return r.GoString() +} + +func (r deployDelayReason) Message(release v1alpha1.Release, applyTime time.Time) string { + if r.IsNoDelay() { + return r.String() + } + + var ( + reasons []string + b strings.Builder + ) + + b.WriteString("Release is ") + + if r.contains(cooldownDelayReason) { + reasons = append(reasons, cooldownDelayMsg) + } + + if r.contains(canaryDelayReason) { + reasons = append(reasons, canaryDelayReasonMsg) + } + + if r.contains(notificationDelayReason) { + reasons = append(reasons, notificationDelayMsg) + } + + if r.contains(outOfWindowReason) { + reasons = append(reasons, outOfWindowMsg) + } + + if r.contains(manualApprovalRequiredReason) { + waitingManualApprovalMsg := fmt.Sprintf(waitingManualApprovalTemplate, v1alpha1.GetReleaseApprovalAnnotation(release)) + reasons = append(reasons, waitingManualApprovalMsg) + } + + if len(reasons) != 0 { + b.WriteString(strings.Join(reasons, ", ")) + + if applyTime.IsZero() { + return b.String() + } + + if r.contains(manualApprovalRequiredReason) { + b.WriteString(". After approval the release will be delayed") + } + + b.WriteString(" until ") + b.WriteString(applyTime.Format(time.RFC822)) + + return b.String() + } + + return r.GoString() +} + +func (r deployDelayReason) add(flag deployDelayReason) deployDelayReason { + return r | flag +} + +func (r deployDelayReason) contains(flag deployDelayReason) bool { + if flag.IsNoDelay() { + return r.IsNoDelay() + } + return r&flag != 0 +} + +func (r deployDelayReason) GoString() string { + reasons := r.splitReasons() + if len(reasons) != 0 { + return strings.Join(reasons, "|") + } + + return fmt.Sprintf("deployDelayReason(0b%b)", byte(r)) +} + +func (r deployDelayReason) splitReasons() []string { + if r.IsNoDelay() { + return []string{"noDelay"} + } + + reasons := make([]string, 0) + + for reason, str := range deployDelayReasonsStr { + if r.contains(reason) { + reasons = append(reasons, str) + } + } + + slices.Sort(reasons) + + return reasons +} diff --git a/deckhouse-controller/pkg/controller/deckhouse-release/updater/deploy_delay_reason_test.go b/deckhouse-controller/pkg/controller/deckhouse-release/updater/deploy_delay_reason_test.go new file mode 100644 index 0000000000..d9a039bc06 --- /dev/null +++ b/deckhouse-controller/pkg/controller/deckhouse-release/updater/deploy_delay_reason_test.go @@ -0,0 +1,61 @@ +/* +Copyright 2024 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package d8updater + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/deckhouse/deckhouse/deckhouse-controller/pkg/apis/deckhouse.io/v1alpha1" + "github.com/deckhouse/deckhouse/go_lib/dependency" +) + +func TestDeployDelayReason(t *testing.T) { + var ( + reason deployDelayReason + deckhouseRelease *v1alpha1.DeckhouseRelease + moduleRelease *v1alpha1.ModuleRelease + zeroTime time.Time + now = dependency.TestDC.GetClock().Now() + ) + + require.True(t, reason == noDelay) + require.True(t, reason.contains(noDelay)) + require.Equal(t, "noDelay", reason.String()) + require.Equal(t, "noDelay", reason.GoString()) + + reason = reason.add(outOfWindowReason) + require.False(t, reason.contains(noDelay)) + require.False(t, reason.contains(manualApprovalRequiredReason)) + require.True(t, reason.contains(outOfWindowReason)) + require.Equal(t, "outOfWindowReason", reason.String()) + require.Equal(t, "Release is waiting for the update window until 17 Oct 19 15:33 UTC", reason.Message(deckhouseRelease, now)) + require.Equal(t, "outOfWindowReason", reason.GoString()) + + reason = reason.add(manualApprovalRequiredReason) + require.True(t, reason.contains(manualApprovalRequiredReason)) + require.True(t, reason.contains(outOfWindowReason)) + require.Equal(t, "manualApprovalRequiredReason, outOfWindowReason", reason.String()) + require.Panics(t, func() { reason.Message(nil, zeroTime) }) + require.Equal(t, "Release is waiting for the update window, waiting for the 'release.deckhouse.io/approved: \"true\"' annotation", reason.Message(deckhouseRelease, zeroTime)) + require.Equal(t, "Release is waiting for the update window, waiting for the 'modules.deckhouse.io/approved: \"true\"' annotation", reason.Message(moduleRelease, zeroTime)) + require.Equal(t, "Release is waiting for the update window, waiting for the 'release.deckhouse.io/approved: \"true\"' annotation. After approval the release will be delayed until 17 Oct 19 15:33 UTC", reason.Message(deckhouseRelease, now)) + require.Equal(t, "Release is waiting for the update window, waiting for the 'modules.deckhouse.io/approved: \"true\"' annotation. After approval the release will be delayed until 17 Oct 19 15:33 UTC", reason.Message(moduleRelease, now)) + require.Equal(t, "manualApprovalRequiredReason|outOfWindowReason", reason.GoString()) +} diff --git a/deckhouse-controller/pkg/controller/deckhouse-release/updater/deploy_time_checker.go b/deckhouse-controller/pkg/controller/deckhouse-release/updater/deploy_time_checker.go new file mode 100644 index 0000000000..93c5f99b38 --- /dev/null +++ b/deckhouse-controller/pkg/controller/deckhouse-release/updater/deploy_time_checker.go @@ -0,0 +1,259 @@ +/* +Copyright 2022 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package d8updater + +import ( + "context" + "fmt" + "log/slog" + "time" + + "github.com/deckhouse/deckhouse/deckhouse-controller/pkg/apis/deckhouse.io/v1alpha1" + "github.com/deckhouse/deckhouse/go_lib/dependency" + "github.com/deckhouse/deckhouse/go_lib/updater" + "github.com/deckhouse/deckhouse/pkg/log" +) + +type DeployTimeChecker struct { + releaseNotifier *ReleaseNotifier + + settings *updater.Settings + + now time.Time + deckhousePodReadyFunc func(ctx context.Context) bool + + logger *log.Logger +} + +func NewDeployTimeChecker(dc dependency.Container, settings *updater.Settings, deckhousePodReadyFunc func(ctx context.Context) bool, logger *log.Logger) *DeployTimeChecker { + return &DeployTimeChecker{ + releaseNotifier: NewReleaseNotifier(settings), + + settings: settings, + + now: dc.GetClock().Now().UTC(), + deckhousePodReadyFunc: deckhousePodReadyFunc, + + logger: logger, + } +} + +type DeployTimeReason struct { + Reason deployDelayReason + Message string + ReleaseApplyAfterTime time.Time + Notified bool +} + +// ProcessPatchReleaseDeployTime +// for patch release we check: +// - No delay from calculated deploy time +func (c *DeployTimeChecker) ProcessPatchReleaseDeployTime(dr *v1alpha1.DeckhouseRelease, res *DeployTimeResult) *DeployTimeReason { + if dr.GetApplyNow() || res.Reason.IsNoDelay() { + return nil + } + + if res.ReleaseApplyTime == c.now { + res.ReleaseApplyTime = time.Time{} + } + + return &DeployTimeReason{ + Message: res.Reason.Message(dr, res.ReleaseApplyTime), + ReleaseApplyAfterTime: res.ReleaseApplyAfterTime, + } +} + +// ProcessMinorReleaseDeployTime +// for minor release we check: +// - Deckhouse pod is ready +// - No delay from calculated deploy time +func (c *DeployTimeChecker) ProcessMinorReleaseDeployTime(ctx context.Context, dr *v1alpha1.DeckhouseRelease, res *DeployTimeResult, dri *ReleaseInfo) *DeployTimeReason { + // check: Deckhouse pod is ready + if !c.deckhousePodReadyFunc(ctx) { + c.logger.Info("Deckhouse is not ready. Skipping upgrade") + + if dri == nil { + return &DeployTimeReason{ + Message: "can not find deployed version, awaiting", + ReleaseApplyAfterTime: res.ReleaseApplyAfterTime, + } + } + + return &DeployTimeReason{ + Message: fmt.Sprintf("awaiting for Deckhouse v%s pod to be ready", dri.Version.String()), + ReleaseApplyAfterTime: res.ReleaseApplyAfterTime, + } + } + + if dr.GetApplyNow() || res.Reason.IsNoDelay() { + return nil + } + + if res.ReleaseApplyTime == c.now { + res.ReleaseApplyTime = time.Time{} + } + + return &DeployTimeReason{ + Message: res.Reason.Message(dr, res.ReleaseApplyTime), + ReleaseApplyAfterTime: res.ReleaseApplyAfterTime, + } +} + +type DeployTimeResult struct { + ReleaseApplyTime time.Time + ReleaseApplyAfterTime time.Time + Reason deployDelayReason +} + +func (c *DeployTimeChecker) checkCanary(dtr *DeployTimeResult, dr *v1alpha1.DeckhouseRelease) { + if dr.GetApplyAfter() != nil { + applyAfter := *dr.GetApplyAfter() + + if c.now.Before(applyAfter) { + c.logger.Warn("release is postponed by canary process, waiting", slog.String("name", dr.GetName())) + + dtr.ReleaseApplyTime = applyAfter + dtr.Reason = dtr.Reason.add(canaryDelayReason) + } + } +} + +func (c *DeployTimeChecker) checkNotify(dtr *DeployTimeResult, dr *v1alpha1.DeckhouseRelease) { + if !dr.GetNotified() && + c.settings.NotificationConfig.MinimalNotificationTime.Duration > 0 { + minApplyTime := c.now.Add(c.settings.NotificationConfig.MinimalNotificationTime.Duration) + + dtr.ReleaseApplyAfterTime = dtr.ReleaseApplyTime + + if !minApplyTime.Before(dtr.ReleaseApplyTime) { + dtr.ReleaseApplyTime = minApplyTime + dtr.ReleaseApplyAfterTime = minApplyTime + dtr.Reason = dtr.Reason.add(notificationDelayReason) + } + } +} + +func (c *DeployTimeChecker) processManualApproved(dtr *DeployTimeResult, dr *v1alpha1.DeckhouseRelease, metricLabels updater.MetricLabels) { + c.logger.Info("release is waiting for manual approval", slog.String("name", dr.GetName())) + + metricLabels.SetTrue(updater.ManualApprovalRequired) + + dtr.ReleaseApplyTime = c.now + dtr.Reason = manualApprovalRequiredReason +} + +func (c *DeployTimeChecker) processWindow(dtr *DeployTimeResult) { + dtr.ReleaseApplyTime = c.settings.Windows.NextAllowedTime(dtr.ReleaseApplyTime) + dtr.Reason = dtr.Reason.add(outOfWindowReason) +} + +func (c *DeployTimeChecker) checkCooldown(dtr *DeployTimeResult, dr *v1alpha1.DeckhouseRelease) { + // check: release cooldown + if dr.GetCooldownUntil() != nil { + cooldownUntil := *dr.GetCooldownUntil() + if c.now.Before(cooldownUntil) { + c.logger.Warn("release in cooldown", slog.String("name", dr.GetName())) + + dtr.ReleaseApplyTime = *dr.GetCooldownUntil() + dtr.Reason = dtr.Reason.add(cooldownDelayReason) + } + } +} + +// CalculatePatchDeployTime calculates deploy time, returns deploy time or postpone time and reason. +// To calculate deploy time, we need to check: +// +// 1) Canary +// 2) Notify +// 3) Window (only in "AutoPatch" mode) +// 4) Manual approve (only in "Manual" mode) +// +// Notify reason must override any other reason +func (c *DeployTimeChecker) CalculatePatchDeployTime(dr *v1alpha1.DeckhouseRelease, metricLabels updater.MetricLabels) *DeployTimeResult { + result := &DeployTimeResult{ + Reason: noDelay, + ReleaseApplyTime: c.now, + } + + if dr.GetApplyNow() { + return result + } + + c.checkCanary(result, dr) + c.checkNotify(result, dr) + + if c.settings.Mode == updater.ModeAutoPatch && !c.settings.Windows.IsAllowed(result.ReleaseApplyTime) { + c.processWindow(result) + } + + if c.settings.Mode == updater.ModeManual && !dr.GetManuallyApproved() { + c.processManualApproved(result, dr, metricLabels) + } + + if !result.ReleaseApplyAfterTime.IsZero() { + result.Reason = notificationDelayReason + + return result + } + + return result +} + +// CalculatePatchDeployTime calculates deploy time, returns deploy time or postpone time and reason. +// To calculate deploy time, we need to check: +// +// 1) Cooldown (TODO: deprecated?) +// 1) Canary (in any mode, except "Manual") +// 2) Notify +// 3) Window (only in "Auto" mode) +// 4) Manual approve (in any mode, except "Auto") +// +// Notify reason must override any other reason +func (c *DeployTimeChecker) CalculateMinorDeployTime(dr *v1alpha1.DeckhouseRelease, metricLabels updater.MetricLabels) *DeployTimeResult { + result := &DeployTimeResult{ + Reason: noDelay, + ReleaseApplyTime: c.now, + } + + if dr.GetApplyNow() { + return result + } + + c.checkCooldown(result, dr) + + if !c.settings.InManualMode() { + c.checkCanary(result, dr) + } + + c.checkNotify(result, dr) + + if c.settings.Mode == updater.ModeAuto && !c.settings.Windows.IsAllowed(result.ReleaseApplyTime) { + c.processWindow(result) + } + + if c.settings.Mode != updater.ModeAuto && !dr.GetManuallyApproved() { + c.processManualApproved(result, dr, metricLabels) + } + + if !result.ReleaseApplyAfterTime.IsZero() { + result.Reason = notificationDelayReason + + return result + } + + return result +} diff --git a/deckhouse-controller/pkg/controller/deckhouse-release/updater/metrics.go b/deckhouse-controller/pkg/controller/deckhouse-release/updater/metrics.go index b9010cc037..da35357408 100644 --- a/deckhouse-controller/pkg/controller/deckhouse-release/updater/metrics.go +++ b/deckhouse-controller/pkg/controller/deckhouse-release/updater/metrics.go @@ -24,21 +24,21 @@ import ( const d8ReleaseBlockedMetricName = "d8_release_info" -func newMetricsUpdater(metricStorage *metricstorage.MetricStorage) *metricsUpdater { - return &metricsUpdater{ +func NewMetricsUpdater(metricStorage *metricstorage.MetricStorage) *MetricsUpdater { + return &MetricsUpdater{ metricStorage: metricStorage, } } -type metricsUpdater struct { +type MetricsUpdater struct { metricStorage *metricstorage.MetricStorage } -func (mu *metricsUpdater) UpdateReleaseMetric(name string, metricLabels updater.MetricLabels) { +func (mu *MetricsUpdater) UpdateReleaseMetric(name string, metricLabels updater.MetricLabels) { mu.PurgeReleaseMetric(name) mu.metricStorage.Grouped().GaugeSet(name, d8ReleaseBlockedMetricName, 1, metricLabels) } -func (mu *metricsUpdater) PurgeReleaseMetric(name string) { +func (mu *MetricsUpdater) PurgeReleaseMetric(name string) { mu.metricStorage.Grouped().ExpireGroupMetricByName(name, d8ReleaseBlockedMetricName) } diff --git a/deckhouse-controller/pkg/controller/deckhouse-release/updater/release.go b/deckhouse-controller/pkg/controller/deckhouse-release/updater/release.go index 344de46eea..7ba9b4316e 100644 --- a/deckhouse-controller/pkg/controller/deckhouse-release/updater/release.go +++ b/deckhouse-controller/pkg/controller/deckhouse-release/updater/release.go @@ -22,6 +22,7 @@ import ( "github.com/deckhouse/deckhouse/deckhouse-controller/pkg/apis/deckhouse.io/v1alpha1" ) +// TODO: replace patch marshalling with controller-runtime patch type StatusPatch v1alpha1.DeckhouseReleaseStatus func (sp StatusPatch) MarshalJSON() ([]byte, error) { diff --git a/deckhouse-controller/pkg/controller/deckhouse-release/updater/release_notifier.go b/deckhouse-controller/pkg/controller/deckhouse-release/updater/release_notifier.go new file mode 100644 index 0000000000..855a9de6b6 --- /dev/null +++ b/deckhouse-controller/pkg/controller/deckhouse-release/updater/release_notifier.go @@ -0,0 +1,166 @@ +/* +Copyright 2022 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package d8updater + +import ( + "bytes" + "context" + "crypto/tls" + "encoding/json" + "fmt" + "net/http" + "time" + + "github.com/deckhouse/deckhouse/deckhouse-controller/pkg/apis/deckhouse.io/v1alpha1" + "github.com/deckhouse/deckhouse/go_lib/updater" +) + +type ReleaseNotifier struct { + settings *updater.Settings +} + +func NewReleaseNotifier(settings *updater.Settings) *ReleaseNotifier { + return &ReleaseNotifier{ + settings: settings, + } +} + +type WebhookData struct { + Subject string `json:"subject"` + Version string `json:"version"` + Requirements map[string]string `json:"requirements,omitempty"` + ChangelogLink string `json:"changelogLink,omitempty"` + + ApplyTime string `json:"applyTime,omitempty"` + Message string `json:"message"` +} + +// SendPatchReleaseNotification sending patch notification (only if notification config has release type "All") +func (u *ReleaseNotifier) SendPatchReleaseNotification(ctx context.Context, dr *v1alpha1.DeckhouseRelease, applyTime time.Time, metricLabels updater.MetricLabels) error { + if dr.GetNotified() { + return nil + } + + if !u.settings.NotificationConfig.IsEmpty() && u.settings.NotificationConfig.ReleaseType == updater.ReleaseTypeAll { + metricLabels.SetFalse(updater.NotificationNotSent) + + err := u.sendReleaseNotification(ctx, dr, applyTime) + if err != nil { + metricLabels.SetTrue(updater.NotificationNotSent) + + return fmt.Errorf("send release notification: %w", err) + } + } + + return nil +} + +func (u *ReleaseNotifier) SendMinorReleaseNotification(ctx context.Context, dr *v1alpha1.DeckhouseRelease, applyTime time.Time, metricLabels updater.MetricLabels) error { + if dr.GetNotified() { + return nil + } + + if !u.settings.NotificationConfig.IsEmpty() { + metricLabels.SetFalse(updater.NotificationNotSent) + + err := u.sendReleaseNotification(ctx, dr, applyTime) + if err != nil { + metricLabels.SetTrue(updater.NotificationNotSent) + + return fmt.Errorf("send release notification: %w", err) + } + } + + return nil +} + +func (u *ReleaseNotifier) sendReleaseNotification(ctx context.Context, dr *v1alpha1.DeckhouseRelease, applyTime time.Time) error { + if u.settings.NotificationConfig.WebhookURL == "" { + return nil + } + + data := &WebhookData{ + Version: dr.GetVersion().String(), + Requirements: dr.GetRequirements(), + ChangelogLink: dr.GetChangelogLink(), + ApplyTime: applyTime.Format(time.RFC3339), + Subject: updater.SubjectDeckhouse, + Message: fmt.Sprintf("New Deckhouse Release %s is available. Release will be applied at: %s", dr.GetVersion().String(), applyTime.Format(time.RFC850)), + } + + err := sendWebhookNotification(ctx, u.settings.NotificationConfig, data) + if err != nil { + return fmt.Errorf("send webhook notification: %w", err) + } + + return nil +} + +func sendWebhookNotification(ctx context.Context, config updater.NotificationConfig, data *WebhookData) error { + client := &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: config.SkipTLSVerify}, + }, + Timeout: 10 * time.Second, + } + + buf := bytes.NewBuffer(nil) + + _, err := retry(5, 2*time.Second, func() (*http.Response, error) { + defer buf.Reset() + + err := json.NewEncoder(buf).Encode(data) + if err != nil { + return nil, err + } + + var req *http.Request + req, err = http.NewRequestWithContext(ctx, http.MethodPost, config.WebhookURL, buf) + if err != nil { + return nil, err + } + + req.Header.Add("Content-Type", "application/json") + config.Auth.Fill(req) + + resp, err := client.Do(req) + if err != nil { + return nil, err + } + + return resp, nil + }) + + return err +} + +func retry[T any](attempts int, sleep time.Duration, f func() (T, error)) (T, error) { + var err error + var result T + + for i := 0; i < attempts; i++ { + result, err = f() + if err == nil { + return result, nil + } + + time.Sleep(sleep) + sleep *= 2 + } + + return result, fmt.Errorf("after %d attempts, last error: %s", attempts, err) +} diff --git a/deckhouse-controller/pkg/controller/deckhouse-release/updater/requirements_checker.go b/deckhouse-controller/pkg/controller/deckhouse-release/updater/requirements_checker.go new file mode 100644 index 0000000000..c143b3f587 --- /dev/null +++ b/deckhouse-controller/pkg/controller/deckhouse-release/updater/requirements_checker.go @@ -0,0 +1,290 @@ +/* +Copyright 2024 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package d8updater + +import ( + "context" + "errors" + "fmt" + + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/yaml" + + "github.com/deckhouse/deckhouse/deckhouse-controller/pkg/apis/deckhouse.io/v1alpha1" + "github.com/deckhouse/deckhouse/go_lib/dependency/extenders" + "github.com/deckhouse/deckhouse/go_lib/dependency/extenders/deckhouseversion" + "github.com/deckhouse/deckhouse/go_lib/dependency/extenders/kubernetesversion" + "github.com/deckhouse/deckhouse/go_lib/dependency/requirements" + "github.com/deckhouse/deckhouse/go_lib/set" + "github.com/deckhouse/deckhouse/go_lib/updater" + "github.com/deckhouse/deckhouse/pkg/log" +) + +const ( + deckhouseClusterConfigurationConfig = "d8-cluster-configuration" + systemNamespace = "kube-system" + k8sAutomaticVersion = "Automatic" +) + +type RequirementsChecker[T any] interface { + MetRequirements(v *T) []NotMetReason +} + +type Check[T any] interface { + GetName() string + Verify(v *T) error +} + +type NotMetReason struct { + Reason string + Message string +} + +var _ RequirementsChecker[v1alpha1.DeckhouseRelease] = (*Checker[v1alpha1.DeckhouseRelease])(nil) + +type Checker[T any] struct { + fns []Check[T] + + logger *log.Logger +} + +func (c *Checker[T]) MetRequirements(v *T) []NotMetReason { + reasons := make([]NotMetReason, 0) + + for _, fn := range c.fns { + err := fn.Verify(v) + if err != nil { + reasons = append(reasons, NotMetReason{ + Reason: fn.GetName(), + Message: err.Error(), + }) + } + } + + return reasons +} + +// NewRequirementsChecker returns DeckhouseRelease checker with this checks: +// +// 1) deckhouse version check +// 2) deckhouse requirements check +// 3) deckhouse kubernetes version check +// +// for more checks information - look at extenders +func NewRequirementsChecker(k8sclient client.Client, enabledModules []string, logger *log.Logger) (*Checker[v1alpha1.DeckhouseRelease], error) { + k8sCheck, err := newKubernetesVersionCheck(k8sclient, enabledModules) + if err != nil { + return nil, err + } + + return &Checker[v1alpha1.DeckhouseRelease]{ + fns: []Check[v1alpha1.DeckhouseRelease]{ + newDeckhouseVersionCheck(enabledModules), + newDeckhouseRequirementsCheck(enabledModules), + k8sCheck, + }, + logger: logger, + }, nil +} + +type deckhouseVersionCheck struct { + name string + + enabledModules set.Set +} + +func newDeckhouseVersionCheck(enabledModules []string) *deckhouseVersionCheck { + return &deckhouseVersionCheck{ + name: "deckhouse version check", + enabledModules: set.New(enabledModules...), + } +} + +func (c *deckhouseVersionCheck) GetName() string { + return c.name +} + +func (c *deckhouseVersionCheck) Verify(dr *v1alpha1.DeckhouseRelease) error { + releaseName, err := deckhouseversion.Instance().ValidateBaseVersion(dr.GetVersion().String()) + if err != nil { + // invalid deckhouse version in deckhouse release + // or an enabled module has requirements + // prevent deckhouse release from becoming predicted + if releaseName == "" || c.enabledModules.Has(releaseName) { + return err + } + } + + return nil +} + +type kubernetesVersionCheck struct { + name string + + enabledModules set.Set + clusterKubernetesVersion string + + k8sclient client.Client +} + +func newKubernetesVersionCheck(k8sclient client.Client, enabledModules []string) (*kubernetesVersionCheck, error) { + c := &kubernetesVersionCheck{ + name: "kubernetes version check", + enabledModules: set.New(enabledModules...), + k8sclient: k8sclient, + } + + err := c.initClusterKubernetesVersion(context.TODO()) + // if discovery failed, we musn't suspend the release + if err != nil { + return nil, fmt.Errorf("getting cluster kubernetes version: %w", err) + } + + return c, nil +} + +func (c *kubernetesVersionCheck) GetName() string { + return c.name +} + +func (c *kubernetesVersionCheck) Verify(dr *v1alpha1.DeckhouseRelease) error { + if c.isKubernetesVersionAutomatic() && len(dr.GetRequirements()["autoK8sVersion"]) > 0 { + if moduleName, err := kubernetesversion.Instance().ValidateBaseVersion(dr.GetRequirements()["autoK8sVersion"]); err != nil { + // invalid auto kubernetes version in deckhouse release + // or an enabled module has requirements + // prevent deckhouse release from becoming predicted + if moduleName == "" || c.enabledModules.Has(moduleName) { + return err + } + } + } + + return nil +} + +func (c *kubernetesVersionCheck) isKubernetesVersionAutomatic() bool { + return c.clusterKubernetesVersion == k8sAutomaticVersion +} + +type clusterConf struct { + KubernetesVersion string `json:"kubernetesVersion"` +} + +func (c *kubernetesVersionCheck) initClusterKubernetesVersion(ctx context.Context) error { + key := client.ObjectKey{Namespace: systemNamespace, Name: deckhouseClusterConfigurationConfig} + secret := new(corev1.Secret) + if err := c.k8sclient.Get(ctx, key, secret); err != nil { + return fmt.Errorf("failed to get secret: %w", err) + } + + clusterConfigurationRaw, ok := secret.Data["cluster-configuration.yaml"] + if !ok { + return fmt.Errorf("expected field 'cluster-configuration.yaml' not found in secret %s", secret.Name) + } + + clusterConf := new(clusterConf) + if err := yaml.Unmarshal(clusterConfigurationRaw, clusterConf); err != nil { + return fmt.Errorf("failed to unmarshal cluster configuration: %w", err) + } + + c.clusterKubernetesVersion = clusterConf.KubernetesVersion + + return nil +} + +type deckhouseRequirementsCheck struct { + name string + + enabledModules set.Set +} + +func newDeckhouseRequirementsCheck(enabledModules []string) *deckhouseRequirementsCheck { + return &deckhouseRequirementsCheck{ + name: "deckhouse requirements check", + enabledModules: set.New(enabledModules...), + } +} + +func (c *deckhouseRequirementsCheck) GetName() string { + return c.name +} + +func (c *deckhouseRequirementsCheck) Verify(dr *v1alpha1.DeckhouseRelease) error { + for key, value := range dr.GetRequirements() { + // these fields are checked by extenders in module release controller + if extenders.IsExtendersField(key) { + continue + } + + passed, err := requirements.CheckRequirement(key, value, c.enabledModules) + if !passed { + msg := fmt.Sprintf("%q requirement for DeckhouseRelease %q not met: %s", key, dr.GetVersion(), err) + if errors.Is(err, requirements.ErrNotRegistered) { + msg = fmt.Sprintf("%q requirement is not registered", key) + } + + return errors.New(msg) + } + } + + return nil +} + +// NewPreApplyChecker returns DeckhouseRelease checker with this checks: +// +// 1) disruption check +func NewPreApplyChecker(settings *updater.Settings, logger *log.Logger) *Checker[v1alpha1.DeckhouseRelease] { + return &Checker[v1alpha1.DeckhouseRelease]{ + fns: []Check[v1alpha1.DeckhouseRelease]{ + newDisruptionCheck(settings), + }, + logger: logger, + } +} + +type disruptionCheck struct { + name string + settings *updater.Settings +} + +// check: release disruptions (hard lock) +func newDisruptionCheck(settings *updater.Settings) *disruptionCheck { + return &disruptionCheck{ + name: "release disruption check", + settings: settings, + } +} + +func (c *disruptionCheck) GetName() string { + return c.name +} + +func (c *disruptionCheck) Verify(dr *v1alpha1.DeckhouseRelease) error { + if !c.settings.InDisruptionApprovalMode() { + return nil + } + + for _, key := range dr.GetDisruptions() { + hasDisruptionUpdate, reason := requirements.HasDisruption(key) + if hasDisruptionUpdate && !dr.GetDisruptionApproved() { + return fmt.Errorf("(`kubectl annotate DeckhouseRelease %s release.deckhouse.io/disruption-approved=true`): %s", dr.GetName(), reason) + } + } + + return nil +} diff --git a/deckhouse-controller/pkg/controller/deckhouse-release/updater/update_task_calculator.go b/deckhouse-controller/pkg/controller/deckhouse-release/updater/update_task_calculator.go new file mode 100644 index 0000000000..31b616bc0d --- /dev/null +++ b/deckhouse-controller/pkg/controller/deckhouse-release/updater/update_task_calculator.go @@ -0,0 +1,257 @@ +/* +Copyright 2024 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package d8updater + +import ( + "context" + "errors" + "fmt" + "slices" + "strings" + + "github.com/Masterminds/semver/v3" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/deckhouse/deckhouse/deckhouse-controller/pkg/apis/deckhouse.io/v1alpha1" + "github.com/deckhouse/deckhouse/pkg/log" +) + +type TaskCalculator struct { + k8sclient client.Client + + log *log.Logger +} + +func NewTaskCalculator(k8sclient client.Client, logger *log.Logger) *TaskCalculator { + return &TaskCalculator{ + k8sclient: k8sclient, + log: logger, + } +} + +type TaskType int + +const ( + Skip TaskType = iota + Await + Process +) + +type Task struct { + TaskType TaskType + Message string + + IsPatch bool + IsSingle bool + IsLatest bool + + DeployedReleaseInfo *ReleaseInfo + QueueDepth int +} + +type ReleaseInfo struct { + Name string + Version *semver.Version +} + +var ErrReleasePhaseIsNotPending = errors.New("release phase is not pending") +var ErrReleaseIsAlreadyDeployed = errors.New("release is already deployed") + +// CalculatePendingReleaseOrder calculate task with information about current reconcile +// +// calculating flow: +// 1) find forced release. if current release has a lower version - skip +// 2) find deployed release. if current release has a lower version - skip +func (p *TaskCalculator) CalculatePendingReleaseOrder(ctx context.Context, dr *v1alpha1.DeckhouseRelease) (*Task, error) { + if dr.GetPhase() != v1alpha1.DeckhouseReleasePhasePending { + return nil, ErrReleasePhaseIsNotPending + } + + releases, err := p.listReleases(ctx) + if err != nil { + return nil, fmt.Errorf("list releases: %w", err) + } + + if len(releases) == 1 { + return &Task{ + TaskType: Process, + IsSingle: true, + IsLatest: true, + }, nil + } + + slices.SortFunc(releases, func(a, b v1alpha1.DeckhouseRelease) int { + return a.GetVersion().Compare(b.GetVersion()) + }) + + forcedReleaseInfo := p.getLatestForcedReleaseInfo(releases) + + // if we have a forced release + if forcedReleaseInfo != nil { + // if forced version is greater than the pending one, this pending release should be skipped + if forcedReleaseInfo.Version.GreaterThan(dr.GetVersion()) { + return &Task{ + TaskType: Skip, + }, nil + } + } + + deployedReleaseInfo := p.getFirstReleaseInfoByPhase(releases, v1alpha1.DeckhouseReleasePhaseDeployed) + + // if we have a deployed release + if deployedReleaseInfo != nil { + // if deployed version is greater than the pending one, this pending release should be skipped + if deployedReleaseInfo.Version.GreaterThan(dr.GetVersion()) { + return &Task{ + TaskType: Skip, + }, nil + } + + // if we patch between reconcile start and calculating + if deployedReleaseInfo.Version.Equal(dr.GetVersion()) { + return nil, ErrReleaseIsAlreadyDeployed + } + } + + releaseIdx, _ := slices.BinarySearchFunc(releases, dr.GetVersion(), func(a v1alpha1.DeckhouseRelease, b *semver.Version) int { + return a.GetVersion().Compare(b) + }) + + releaseQueueDepth := len(releases) - 1 - releaseIdx + isLatestRelease := releaseQueueDepth == 0 + isPatch := true + + // check previous release + // only for awaiting purpose + if releaseIdx > 0 { + prevRelease := releases[releaseIdx-1] + + // if release version is greater in major or minor version than previous release + if dr.GetVersion().Major() > prevRelease.GetVersion().Major() || + dr.GetVersion().Minor() > prevRelease.GetVersion().Minor() { + isPatch = false + + // it must await if previous release has Deployed state + if prevRelease.GetPhase() != v1alpha1.DeckhouseReleasePhaseDeployed { + msg := prevRelease.Status.Message + if !strings.Contains(msg, "awaiting") { + msg = fmt.Sprintf("awaiting for v%s release to be deployed", prevRelease.GetVersion().String()) + } + + return &Task{ + TaskType: Await, + Message: msg, + DeployedReleaseInfo: deployedReleaseInfo, + }, nil + } + + // it must await if deployed release has minor version more than one + if deployedReleaseInfo != nil && dr.GetVersion().Minor()-1 > deployedReleaseInfo.Version.Minor() { + return &Task{ + TaskType: Await, + Message: fmt.Sprintf("minor version is more than deployed v%s by one", prevRelease.GetVersion().String()), + DeployedReleaseInfo: deployedReleaseInfo, + }, nil + } + } + } + + // check next release + // patch calculate logic + if len(releases)-1 > releaseIdx { + nextRelease := releases[releaseIdx+1] + + // if nextRelease version is greater in major or minor version + // current release is definitely greatest at patch version + // + // "isPatch" value could be false, if we have versions like: + // 1.65.0 (Deployed) + // 1.66.0 (Pending) - is greatest patch now, bot must handle like minor version bump + // 1.67.0 (Pending) + if dr.GetVersion().Major() < nextRelease.GetVersion().Major() || + dr.GetVersion().Minor() < nextRelease.GetVersion().Minor() { + return &Task{ + TaskType: Process, + IsPatch: isPatch, + IsLatest: isLatestRelease, + DeployedReleaseInfo: deployedReleaseInfo, + QueueDepth: releaseQueueDepth, + }, nil + } + + return &Task{ + TaskType: Skip, + IsPatch: isPatch, + }, nil + } + + // neighbours checks passed + // only minor/major releases must be here + return &Task{ + TaskType: Process, + IsLatest: isLatestRelease, + IsPatch: isPatch, + DeployedReleaseInfo: deployedReleaseInfo, + QueueDepth: releaseQueueDepth, + }, nil +} + +func (p *TaskCalculator) listReleases(ctx context.Context) ([]v1alpha1.DeckhouseRelease, error) { + var releases v1alpha1.DeckhouseReleaseList + err := p.k8sclient.List(ctx, &releases) + if err != nil { + return nil, fmt.Errorf("get deckhouse releases: %w", err) + } + + return releases.Items, nil +} + +// getFirstReleaseInfoByPhase +// releases slice must be sorted asc +func (p *TaskCalculator) getFirstReleaseInfoByPhase(releases []v1alpha1.DeckhouseRelease, phase string) *ReleaseInfo { + idx := slices.IndexFunc(releases, func(a v1alpha1.DeckhouseRelease) bool { + return a.Status.Phase == phase + }) + + if idx == -1 { + return nil + } + + filteredDR := releases[idx] + + return &ReleaseInfo{ + Name: filteredDR.GetName(), + Version: filteredDR.GetVersion(), + } +} + +// getLatestForcedReleaseInfo +// releases slice must be sorted asc +func (p *TaskCalculator) getLatestForcedReleaseInfo(releases []v1alpha1.DeckhouseRelease) *ReleaseInfo { + for _, dr := range slices.Backward(releases) { + if !dr.GetForce() { + continue + } + + return &ReleaseInfo{ + Name: dr.GetName(), + Version: dr.GetVersion(), + } + } + + return nil +} diff --git a/deckhouse-controller/pkg/controller/deckhouse-release/updater/updater.go b/deckhouse-controller/pkg/controller/deckhouse-release/updater/updater.go deleted file mode 100644 index 4d4d57e7c8..0000000000 --- a/deckhouse-controller/pkg/controller/deckhouse-release/updater/updater.go +++ /dev/null @@ -1,209 +0,0 @@ -/* -Copyright 2022 Flant JSC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package d8updater - -import ( - "context" - "encoding/json" - "fmt" - "strconv" - "time" - - metricstorage "github.com/flant/shell-operator/pkg/metric_storage" - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/yaml" - - "github.com/deckhouse/deckhouse/deckhouse-controller/pkg/apis/deckhouse.io/v1alpha1" - "github.com/deckhouse/deckhouse/go_lib/dependency" - "github.com/deckhouse/deckhouse/go_lib/updater" - "github.com/deckhouse/deckhouse/pkg/log" -) - -const ( - IsUpdatingAnnotation = "release.deckhouse.io/isUpdating" - NotifiedAnnotation = "release.deckhouse.io/notified" - - deckhouseClusterConfigurationConfig = "d8-cluster-configuration" - systemNamespace = "kube-system" - k8sAutomaticVersion = "Automatic" -) - -func NewDeckhouseUpdater( - ctx context.Context, - logger *log.Logger, - client client.Client, - dc dependency.Container, - updateSettings *updater.Settings, - releaseData updater.DeckhouseReleaseData, - metricStorage *metricstorage.MetricStorage, - podIsReady, - clusterBootstrapping bool, - imagesRegistry string, - enabledModules []string, -) *updater.Updater[*v1alpha1.DeckhouseRelease] { - return updater.NewUpdater[*v1alpha1.DeckhouseRelease]( - ctx, - dc, - logger, - updateSettings, - releaseData, - podIsReady, - clusterBootstrapping, - NewKubeAPI(client, dc, imagesRegistry), - newMetricsUpdater(metricStorage), - newWebhookDataSource(logger), - enabledModules, - ) -} - -func newWebhookDataSource(logger *log.Logger) *webhookDataSource { - return &webhookDataSource{logger: logger} -} - -type webhookDataSource struct { - logger *log.Logger -} - -func (s *webhookDataSource) Fill(output *updater.WebhookData, _ *v1alpha1.DeckhouseRelease, applyTime time.Time) { - if output == nil { - s.logger.Error("webhook data must be defined") - return - } - - output.Subject = updater.SubjectDeckhouse - output.Message = fmt.Sprintf("New Deckhouse Release %s is available. Release will be applied at: %s", output.Version, applyTime.Format(time.RFC850)) -} - -func NewKubeAPI(client client.Client, dc dependency.Container, imagesRegistry string) *KubeAPI { - return &KubeAPI{client: client, dc: dc, imagesRegistry: imagesRegistry} -} - -type KubeAPI struct { - client client.Client - dc dependency.Container - imagesRegistry string -} - -func (api *KubeAPI) UpdateReleaseStatus(ctx context.Context, release *v1alpha1.DeckhouseRelease, msg, phase string) error { - // reset transition time only if phase changes - if release.Status.Phase != phase { - release.Status.TransitionTime = metav1.NewTime(api.dc.GetClock().Now().UTC()) - } - release.Status.Phase = phase - release.Status.Message = msg - - return api.client.Status().Update(ctx, release) -} - -func (api *KubeAPI) PatchReleaseAnnotations(ctx context.Context, release *v1alpha1.DeckhouseRelease, annotations map[string]any) error { - patch, _ := json.Marshal(map[string]any{ - "metadata": map[string]any{ - "annotations": annotations, - }, - }) - - p := client.RawPatch(types.MergePatchType, patch) - return api.client.Patch(ctx, release, p) -} - -func (api *KubeAPI) PatchReleaseApplyAfter(release *v1alpha1.DeckhouseRelease, applyTime time.Time) error { - ctx := context.Background() - patch, _ := json.Marshal(map[string]interface{}{ - "spec": map[string]interface{}{ - "applyAfter": applyTime, - }, - "metadata": map[string]interface{}{ - "annotations": map[string]string{ - "release.deckhouse.io/notification-time-shift": "true", - }, - }, - }) - - p := client.RawPatch(types.MergePatchType, patch) - return api.client.Patch(ctx, release, p) -} - -func (api *KubeAPI) DeployRelease(ctx context.Context, release *v1alpha1.DeckhouseRelease) error { - key := client.ObjectKey{Namespace: "d8-system", Name: "deckhouse"} - var depl appsv1.Deployment - err := api.client.Get(ctx, key, &depl) - if err != nil { - return fmt.Errorf("get deployment %s: %w", key, err) - } - - // patch deckhouse deployment is faster than set internal values and then upgrade by helm - // we can set "deckhouse.internal.currentReleaseImageName" value but lets left it this way - depl.Spec.Template.Spec.Containers[0].Image = api.imagesRegistry + ":" + release.Spec.Version - - // dryrun - if val, ok := release.GetAnnotations()["dryrun"]; ok && val == "true" { - // TODO: write log about dry run - go func() { - time.Sleep(3 * time.Second) - var releases v1alpha1.DeckhouseReleaseList - err = api.client.List(ctx, &releases) - if err != nil { - return - } - - for _, r := range releases.Items { - if r.GetName() == release.GetName() { - continue - } - if r.Status.Phase != v1alpha1.ModuleReleasePhasePending { - continue - } - // patch releases to trigger their requeue - _ = api.PatchReleaseAnnotations(ctx, &r, map[string]any{"triggered_by_dryrun": release.GetName()}) - } - }() - return nil - } - - return api.client.Update(ctx, &depl) -} - -func (api *KubeAPI) SaveReleaseData(ctx context.Context, release *v1alpha1.DeckhouseRelease, data updater.DeckhouseReleaseData) error { - return api.PatchReleaseAnnotations(ctx, release, map[string]interface{}{ - IsUpdatingAnnotation: strconv.FormatBool(data.IsUpdating), - NotifiedAnnotation: strconv.FormatBool(data.Notified), - }) -} - -func (api *KubeAPI) IsKubernetesVersionAutomatic(ctx context.Context) (bool, error) { - key := client.ObjectKey{Namespace: systemNamespace, Name: deckhouseClusterConfigurationConfig} - secret := new(corev1.Secret) - if err := api.client.Get(ctx, key, secret); err != nil { - return false, fmt.Errorf("check kubernetes version: failed to get secret: %w", err) - } - - var clusterConf struct { - KubernetesVersion string `json:"kubernetesVersion"` - } - clusterConfigurationRaw, ok := secret.Data["cluster-configuration.yaml"] - if !ok { - return false, fmt.Errorf("check kubernetes version: expected field 'cluster-configuration.yaml' not found in secret %s", secret.Name) - } - if err := yaml.Unmarshal(clusterConfigurationRaw, &clusterConf); err != nil { - return false, fmt.Errorf("check kubernetes version: failed to unmarshal cluster configuration: %w", err) - } - return clusterConf.KubernetesVersion == k8sAutomaticVersion, nil -} diff --git a/deckhouse-controller/pkg/controller/module-controllers/release/updater.go b/deckhouse-controller/pkg/controller/module-controllers/release/updater.go index 2fc3122aa7..be5a20f8e2 100644 --- a/deckhouse-controller/pkg/controller/module-controllers/release/updater.go +++ b/deckhouse-controller/pkg/controller/module-controllers/release/updater.go @@ -126,8 +126,8 @@ func (k *kubeAPI) PatchReleaseAnnotations(ctx context.Context, release *v1alpha1 func (k *kubeAPI) PatchReleaseApplyAfter(release *v1alpha1.ModuleRelease, applyTime time.Time) error { return k.PatchReleaseAnnotations(context.TODO(), release, map[string]any{ - "release.deckhouse.io/notification-time-shift": "true", - "release.deckhouse.io/applyAfter": applyTime.Format(time.RFC3339), + v1alpha1.DeckhouseReleaseAnnotationNotificationTimeShift: "true", + v1alpha1.DeckhouseReleaseAnnotationApplyAfter: applyTime.Format(time.RFC3339), }) } @@ -222,7 +222,7 @@ func (k *kubeAPI) SaveReleaseData(ctx context.Context, release *v1alpha1.ModuleR return k.PatchReleaseAnnotations(ctx, release, map[string]interface{}{ // "release.deckhouse.io/isUpdating": strconv.FormatBool(data.IsUpdating), // I don't think we need this flag for ModuleReleases - "release.deckhouse.io/notified": strconv.FormatBool(data.Notified), + v1alpha1.DeckhouseReleaseAnnotationNotified: strconv.FormatBool(data.Notified), }) } diff --git a/deckhouse-controller/pkg/controller/module-controllers/utils/utils.go b/deckhouse-controller/pkg/controller/module-controllers/utils/utils.go index 0621a9c8b2..bac4fdadfb 100644 --- a/deckhouse-controller/pkg/controller/module-controllers/utils/utils.go +++ b/deckhouse-controller/pkg/controller/module-controllers/utils/utils.go @@ -23,6 +23,7 @@ import ( "os" "path/filepath" "regexp" + "strconv" "strings" "github.com/gofrs/uuid/v5" @@ -89,13 +90,19 @@ func GenerateRegistryOptions(ri *RegistryConfig, logger *log.Logger) []cr.Option type DeckhouseRegistrySecret struct { DockerConfig string Address string - ClusterIsBootstrapped string + ClusterIsBootstrapped bool ImageRegistry string Path string Scheme string CA string } +var ErrDockerConfigFieldIsNotFound = errors.New("secret has no .dockerconfigjson field") +var ErrAddressFieldIsNotFound = errors.New("secret has no address field") +var ErrClusterIsBootstrappedFieldIsNotFound = errors.New("secret has no clusterIsBootstrapped field") +var ErrImageRegistryFieldIsNotFound = errors.New("secret has no imagesRegistry field") +var ErrPathFieldIsNotFound = errors.New("secret has no path field") +var ErrSchemeFieldIsNotFound = errors.New("secret has no scheme field") var ErrCAFieldIsNotFound = errors.New("secret has no ca field") func ParseDeckhouseRegistrySecret(data map[string][]byte) (*DeckhouseRegistrySecret, error) { @@ -103,32 +110,43 @@ func ParseDeckhouseRegistrySecret(data map[string][]byte) (*DeckhouseRegistrySec dockerConfig, ok := data[".dockerconfigjson"] if !ok { - err = errors.Join(err, errors.New("secret has no .dockerconfigjson field")) + err = errors.Join(err, ErrDockerConfigFieldIsNotFound) } address, ok := data["address"] if !ok { - err = errors.Join(err, errors.New("secret has no address field")) + err = errors.Join(err, ErrAddressFieldIsNotFound) } - clusterIsBootstrapped, ok := data["clusterIsBootstrapped"] + clusterIsBootstrappedRaw, ok := data["clusterIsBootstrapped"] if !ok { - err = errors.Join(err, errors.New("secret has no clusterIsBootstrapped field")) + err = errors.Join(err, ErrClusterIsBootstrappedFieldIsNotFound) + } + + var clusterIsBootstrapped bool + var castErr error + if ok { + trimmedBool := strings.ReplaceAll(string(clusterIsBootstrappedRaw), "\"", "") + + clusterIsBootstrapped, castErr = strconv.ParseBool(trimmedBool) + if castErr != nil { + err = errors.Join(err, fmt.Errorf("clusterIsBootstrapped is not bool: %w", castErr)) + } } imagesRegistry, ok := data["imagesRegistry"] if !ok { - err = errors.Join(err, errors.New("secret has no imagesRegistry field")) + err = errors.Join(err, ErrImageRegistryFieldIsNotFound) } path, ok := data["path"] if !ok { - err = errors.Join(err, errors.New("secret has no path field")) + err = errors.Join(err, ErrPathFieldIsNotFound) } scheme, ok := data["scheme"] if !ok { - err = errors.Join(err, errors.New("secret has no scheme field")) + err = errors.Join(err, ErrSchemeFieldIsNotFound) } ca, ok := data["ca"] @@ -139,7 +157,7 @@ func ParseDeckhouseRegistrySecret(data map[string][]byte) (*DeckhouseRegistrySec return &DeckhouseRegistrySecret{ DockerConfig: string(dockerConfig), Address: string(address), - ClusterIsBootstrapped: string(clusterIsBootstrapped), + ClusterIsBootstrapped: clusterIsBootstrapped, ImageRegistry: string(imagesRegistry), Path: string(path), Scheme: string(scheme), diff --git a/deckhouse-controller/pkg/registry/registry.go b/deckhouse-controller/pkg/registry/registry.go index 6d84e98af5..e5654d4bfe 100644 --- a/deckhouse-controller/pkg/registry/registry.go +++ b/deckhouse-controller/pkg/registry/registry.go @@ -341,7 +341,10 @@ func getDeckhouseRegistry(ctx context.Context) (string, string, *utils.RegistryC return "", "", nil, fmt.Errorf("list ModuleSource got an error: %w", err) } - drs, _ := utils.ParseDeckhouseRegistrySecret(secret.Data) + drs, err := utils.ParseDeckhouseRegistrySecret(secret.Data) + if errors.Is(err, utils.ErrImageRegistryFieldIsNotFound) { + drs.ImageRegistry = drs.Address + drs.Path + } var discoverySecret corev1.Secret key := types.NamespacedName{Namespace: "d8-system", Name: "deckhouse-discovery"} diff --git a/go_lib/updater/error.go b/go_lib/updater/error.go index 7ca812b143..1e2c5cdf5c 100644 --- a/go_lib/updater/error.go +++ b/go_lib/updater/error.go @@ -16,19 +16,14 @@ limitations under the License. package updater -import ( - "time" -) +var ErrDeployConditionsNotMet = NewNotReadyForDeployError("deploy conditions not met") -var ErrDeployConditionsNotMet = NewNotReadyForDeployError("deploy conditions not met", 0) - -func NewNotReadyForDeployError(message string, retryDelay time.Duration) *NotReadyForDeployError { - return &NotReadyForDeployError{message: message, retryDelay: retryDelay} +func NewNotReadyForDeployError(message string) *NotReadyForDeployError { + return &NotReadyForDeployError{message: message} } type NotReadyForDeployError struct { - message string - retryDelay time.Duration + message string } func (n *NotReadyForDeployError) Error() string { @@ -39,7 +34,3 @@ func (n *NotReadyForDeployError) Error() string { return message } - -func (n *NotReadyForDeployError) RetryDelay() time.Duration { - return n.retryDelay -} diff --git a/go_lib/updater/metrics.go b/go_lib/updater/metrics.go index 1b5476fe5c..a736054437 100644 --- a/go_lib/updater/metrics.go +++ b/go_lib/updater/metrics.go @@ -27,13 +27,16 @@ const ( ) func NewReleaseMetricLabels(release v1alpha1.Release) MetricLabels { - labels := make(map[string]string, 6) - labels[ManualApprovalRequired] = "false" - labels[DisruptionApprovalRequired] = "false" - labels[RequirementsNotMet] = "false" - labels[ReleaseQueueDepth] = "nil" + labels := make(MetricLabels, 6) + labels["name"] = release.GetName() - labels[NotificationNotSent] = "false" + + labels.SetFalse(ManualApprovalRequired) + labels.SetFalse(DisruptionApprovalRequired) + labels.SetFalse(RequirementsNotMet) + labels.SetFalse(NotificationNotSent) + + labels[ReleaseQueueDepth] = "nil" if _, ok := release.(*v1alpha1.ModuleRelease); ok { labels["moduleName"] = release.GetModuleName() @@ -42,7 +45,15 @@ func NewReleaseMetricLabels(release v1alpha1.Release) MetricLabels { return labels } -type MetricsUpdater[R v1alpha1.Release] interface { +func (ml MetricLabels) SetTrue(key string) { + ml[key] = "true" +} + +func (ml MetricLabels) SetFalse(key string) { + ml[key] = "false" +} + +type MetricsUpdater interface { UpdateReleaseMetric(string, MetricLabels) PurgeReleaseMetric(string) } diff --git a/go_lib/updater/notification.go b/go_lib/updater/notification.go index ac5b986bd9..75cfa9878a 100644 --- a/go_lib/updater/notification.go +++ b/go_lib/updater/notification.go @@ -41,6 +41,10 @@ type NotificationConfig struct { ReleaseType ReleaseType `json:"releaseType"` } +func (cfg *NotificationConfig) IsEmpty() bool { + return cfg != nil && *cfg == NotificationConfig{} +} + type Auth struct { Basic *BasicAuth `json:"basic,omitempty"` Token *string `json:"bearerToken,omitempty"` diff --git a/go_lib/updater/settings.go b/go_lib/updater/settings.go index f3bacfff7d..d3d902f661 100644 --- a/go_lib/updater/settings.go +++ b/go_lib/updater/settings.go @@ -22,3 +22,15 @@ type Settings struct { Mode UpdateMode Windows update.Windows } + +func (s *Settings) InDisruptionApprovalMode() bool { + if s.DisruptionApprovalMode == "" || s.DisruptionApprovalMode == "Auto" { + return false + } + + return true +} + +func (s *Settings) InManualMode() bool { + return s.Mode == ModeManual +} diff --git a/go_lib/updater/updater.go b/go_lib/updater/updater.go index 8d24aab44e..29465f512f 100644 --- a/go_lib/updater/updater.go +++ b/go_lib/updater/updater.go @@ -63,7 +63,7 @@ type Updater[R v1alpha1.Release] struct { logger *log.Logger kubeAPI KubeAPI[R] - metricsUpdater MetricsUpdater[R] + metricsUpdater MetricsUpdater webhookDataSource WebhookDataSource[R] // don't modify releases order, logic is based on this sorted slice @@ -76,7 +76,8 @@ type Updater[R v1alpha1.Release] struct { deckhousePodIsReady bool deckhouseIsBootstrapping bool - releaseData DeckhouseReleaseData + + releaseData DeckhouseReleaseData } func NewUpdater[R v1alpha1.Release]( @@ -87,7 +88,7 @@ func NewUpdater[R v1alpha1.Release]( data DeckhouseReleaseData, podIsReady, isBootstrapping bool, kubeAPI KubeAPI[R], - metricsUpdater MetricsUpdater[R], + metricsUpdater MetricsUpdater, webhookDataSource WebhookDataSource[R], enabledModules []string, ) *Updater[R] { @@ -428,7 +429,7 @@ func (u *Updater[R]) checkReleaseDisruptions(rl R) bool { hasDisruptionUpdate, reason := requirements.HasDisruption(key) if hasDisruptionUpdate { if !rl.GetDisruptionApproved() { - msg := fmt.Sprintf("Release requires disruption approval (`kubectl annotate DeckhouseRelease %s release.deckhouse.io/disruption-approved=true`): %s", rl.GetName(), reason) + msg := fmt.Sprintf("release requires disruption approval (`kubectl annotate DeckhouseRelease %s release.deckhouse.io/disruption-approved=true`): %s", rl.GetName(), reason) err := u.updateStatus(rl, msg, PhasePending) if err != nil { u.logger.Error("update status", log.Err(err)) @@ -489,7 +490,7 @@ func (u *Updater[R]) runReleaseDeploy(predictedRelease R, currentRelease *R) err ctx, predictedRelease, map[string]interface{}{ - "release.deckhouse.io/apply-now": nil, + v1alpha1.DeckhouseReleaseAnnotationApplyNow: nil, }) if err != nil { return fmt.Errorf("remove apply-now annotation: %w", err) @@ -518,13 +519,17 @@ func (u *Updater[R]) PredictNextRelease(release R) { } } + // TODO: remove double loop??? for i, rl := range u.releases { switch rl.GetPhase() { case PhaseSuperseded, PhaseSuspended, PhaseSkipped: // pass case PhasePending: + // TODO: get check result and update release within releaseRequirementsMet := u.checkReleaseRequirements(rl) + + // note: here's we have assignment of predicted release u.processPendingRelease(i, rl, releaseRequirementsMet) // update metric only for the release that initiated prediction so as not to provoke metrics churn on every prediction if rl.GetName() == release.GetName() { @@ -574,7 +579,7 @@ func (u *Updater[R]) ApplyForcedRelease(ctx context.Context) error { // remove annotation err := u.kubeAPI.PatchReleaseAnnotations(ctx, forcedRelease, map[string]any{ - "release.deckhouse.io/force": nil, + v1alpha1.DeckhouseReleaseAnnotationForce: nil, }) if err != nil { return fmt.Errorf("patch force annotation: %w", err) @@ -813,17 +818,13 @@ func (u *Updater[R]) postponeDeploy(release R, reason deployDelayReason, applyTi var ( zeroTime time.Time - retryDelay time.Duration statusMessage string ) - if !applyTime.IsZero() { - retryDelay = applyTime.Sub(u.now) - } - if applyTime == u.now { applyTime = zeroTime } + statusMessage = reason.Message(release, applyTime) err := u.updateStatus(release, statusMessage, PhasePending) @@ -831,5 +832,5 @@ func (u *Updater[R]) postponeDeploy(release R, reason deployDelayReason, applyTi return fmt.Errorf("update release %s status: %w", release.GetName(), err) } - return NewNotReadyForDeployError(statusMessage, retryDelay) + return NewNotReadyForDeployError(statusMessage) }