Skip to content

Commit a42be1a

Browse files
everettraveneverettraven
committed
Add generic crd upgrade safety preflight check
and some initial validations for handling scope changes and removal of existing stored versions Signed-off-by: everettraven <everettraven@gmail.com>
1 parent 88625c2 commit a42be1a

37 files changed

+17136
-6
lines changed

‎go.mod

+2-1
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,10 @@ require (
1919
golang.org/x/net v0.22.0
2020
gopkg.in/yaml.v2 v2.4.0
2121
k8s.io/api v0.29.3
22+
k8s.io/apiextensions-apiserver v0.29.3
2223
k8s.io/apimachinery v0.29.3
2324
k8s.io/client-go v0.29.3
24-
k8s.io/component-helpers v0.29.1
25+
k8s.io/component-helpers v0.29.3
2526
sigs.k8s.io/yaml v1.4.0
2627
)
2728

‎go.sum

+6-4
Original file line numberDiff line numberDiff line change
@@ -62,8 +62,8 @@ github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQL
6262
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
6363
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
6464
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
65-
github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
66-
github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
65+
github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
66+
github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
6767
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
6868
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
6969
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
@@ -510,12 +510,14 @@ honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWh
510510
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
511511
k8s.io/api v0.29.3 h1:2ORfZ7+bGC3YJqGpV0KSDDEVf8hdGQ6A03/50vj8pmw=
512512
k8s.io/api v0.29.3/go.mod h1:y2yg2NTyHUUkIoTC+phinTnEa3KFM6RZ3szxt014a80=
513+
k8s.io/apiextensions-apiserver v0.29.3 h1:9HF+EtZaVpFjStakF4yVufnXGPRppWFEQ87qnO91YeI=
514+
k8s.io/apiextensions-apiserver v0.29.3/go.mod h1:po0XiY5scnpJfFizNGo6puNU6Fq6D70UJY2Cb2KwAVc=
513515
k8s.io/apimachinery v0.29.3 h1:2tbx+5L7RNvqJjn7RIuIKu9XTsIZ9Z5wX2G22XAa5EU=
514516
k8s.io/apimachinery v0.29.3/go.mod h1:hx/S4V2PNW4OMg3WizRrHutyB5la0iCUbZym+W0EQIU=
515517
k8s.io/client-go v0.29.3 h1:R/zaZbEAxqComZ9FHeQwOh3Y1ZUs7FaHKZdQtIc2WZg=
516518
k8s.io/client-go v0.29.3/go.mod h1:tkDisCvgPfiRpxGnOORfkljmS+UrW+WtXAy2fTvXJB0=
517-
k8s.io/component-helpers v0.29.1 h1:54MMEDu6xeJmMtAKztsPwu0kJKr4+jCUzaEIn2UXRoc=
518-
k8s.io/component-helpers v0.29.1/go.mod h1:+I7xz4kfUgxWAPJIVKrqe4ml4rb9UGpazlOmhXYo+cY=
519+
k8s.io/component-helpers v0.29.3 h1:1dqZswuZgT2ZMixYeORyCUOAApXxgsvjVSgfoUT+P4o=
520+
k8s.io/component-helpers v0.29.3/go.mod h1:yiDqbRQrnQY+sPju/bL7EkwDJb6LVOots53uZNMZBos=
519521
k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0=
520522
k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo=
521523
k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780=

‎pkg/kapp/cmd/kapp.go

+2
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import (
1616
cmdcore "github.com/vmware-tanzu/carvel-kapp/pkg/kapp/cmd/core"
1717
cmdsa "github.com/vmware-tanzu/carvel-kapp/pkg/kapp/cmd/serviceaccount"
1818
cmdtools "github.com/vmware-tanzu/carvel-kapp/pkg/kapp/cmd/tools"
19+
"github.com/vmware-tanzu/carvel-kapp/pkg/kapp/crdupgradesafety"
1920
"github.com/vmware-tanzu/carvel-kapp/pkg/kapp/logger"
2021
"github.com/vmware-tanzu/carvel-kapp/pkg/kapp/permissions"
2122
"github.com/vmware-tanzu/carvel-kapp/pkg/kapp/preflight"
@@ -57,6 +58,7 @@ func NewDefaultKappCmd(ui *ui.ConfUI) *cobra.Command {
5758
func defaultKappPreflightRegistry(depsFactory cmdcore.DepsFactory) *preflight.Registry {
5859
registry := preflight.NewRegistry(map[string]preflight.Check{
5960
"PermissionValidation": permissions.NewPreflight(depsFactory, false),
61+
"CRDUpgradeSafety": crdupgradesafety.NewPreflight(depsFactory, false),
6062
})
6163

6264
return registry

‎pkg/kapp/crdupgradesafety/preflight.go

+109
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
// Copyright 2024 VMware, Inc.
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
package crdupgradesafety
5+
6+
import (
7+
"context"
8+
"errors"
9+
"fmt"
10+
11+
cmdcore "github.com/vmware-tanzu/carvel-kapp/pkg/kapp/cmd/core"
12+
ctldgraph "github.com/vmware-tanzu/carvel-kapp/pkg/kapp/diffgraph"
13+
"github.com/vmware-tanzu/carvel-kapp/pkg/kapp/preflight"
14+
v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
15+
apierrors "k8s.io/apimachinery/pkg/api/errors"
16+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
17+
"k8s.io/apimachinery/pkg/runtime"
18+
)
19+
20+
var _ preflight.Check = (*Preflight)(nil)
21+
22+
// Preflight is an implementation of preflight.Check
23+
// to make it easier to add crd upgrade validation
24+
// as a preflight check
25+
type Preflight struct {
26+
depsFactory cmdcore.DepsFactory
27+
enabled bool
28+
validator *Validator
29+
}
30+
31+
func NewPreflight(df cmdcore.DepsFactory, enabled bool) *Preflight {
32+
return &Preflight{
33+
depsFactory: df,
34+
enabled: enabled,
35+
validator: &Validator{
36+
Validations: []Validation{
37+
NewValidationFunc("NoScopeChange", NoScopeChange),
38+
NewValidationFunc("NoStoredVersionRemoved", NoStoredVersionRemoved),
39+
},
40+
},
41+
}
42+
}
43+
44+
func (p *Preflight) Enabled() bool {
45+
return p.enabled
46+
}
47+
48+
func (p *Preflight) SetEnabled(enabled bool) {
49+
p.enabled = enabled
50+
}
51+
52+
func (p *Preflight) Run(ctx context.Context, changeGraph *ctldgraph.ChangeGraph) error {
53+
dCli, err := p.depsFactory.DynamicClient(cmdcore.DynamicClientOpts{})
54+
if err != nil {
55+
return fmt.Errorf("getting dynamic client: %w", err)
56+
}
57+
crdCli := dCli.Resource(v1.SchemeGroupVersion.WithResource("customresourcedefinitions"))
58+
59+
validateErrs := []error{}
60+
for _, change := range changeGraph.All() {
61+
// Loop through all the changes looking for "upsert" operations on
62+
// a CRD. "upsert" is used for create + update operations
63+
if change.Change.Op() != ctldgraph.ActualChangeOpUpsert {
64+
continue
65+
}
66+
res := change.Change.Resource()
67+
if res.GroupVersion().WithKind(res.Kind()) != v1.SchemeGroupVersion.WithKind("CustomResourceDefinition") {
68+
continue
69+
}
70+
71+
// to properly determine if this is an update operation, attempt to fetch
72+
// the "old" CRD from the cluster
73+
uOldCRD, err := crdCli.Get(ctx, res.Name(), metav1.GetOptions{})
74+
if err != nil {
75+
// if the resource is not found, this "upsert" operation
76+
// translates to a "create" request being made. Skip this change
77+
if apierrors.IsNotFound(err) {
78+
continue
79+
}
80+
81+
return fmt.Errorf("checking for existing CRD resource: %w", err)
82+
}
83+
84+
oldCRD := &v1.CustomResourceDefinition{}
85+
s := runtime.NewScheme()
86+
if err := v1.AddToScheme(s); err != nil {
87+
return fmt.Errorf("adding apiextension apis to scheme: %w", err)
88+
}
89+
if err := s.Convert(uOldCRD, oldCRD, nil); err != nil {
90+
return fmt.Errorf("couldn't convert old CRD resource to a CRD object: %w", err)
91+
}
92+
93+
newCRD := &v1.CustomResourceDefinition{}
94+
if err := res.AsUncheckedTypedObj(newCRD); err != nil {
95+
return fmt.Errorf("couldn't convert new CRD resource to a CRD object: %w", err)
96+
}
97+
98+
if err = p.validator.Validate(*oldCRD, *newCRD); err != nil {
99+
validateErrs = append(validateErrs, err)
100+
}
101+
}
102+
103+
if len(validateErrs) > 0 {
104+
baseErr := errors.New("validation for safe CRD upgrades failed")
105+
return errors.Join(append([]error{baseErr}, validateErrs...)...)
106+
}
107+
108+
return nil
109+
}

‎pkg/kapp/crdupgradesafety/validator.go

+93
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
// Copyright 2024 VMware, Inc.
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
package crdupgradesafety
5+
6+
import (
7+
"errors"
8+
"fmt"
9+
10+
v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
11+
"k8s.io/apimachinery/pkg/util/sets"
12+
)
13+
14+
// Validation is a representation of a validation to run
15+
// against a CRD being upgraded
16+
type Validation interface {
17+
// Validate contains the actual validation logic. An error being
18+
// returned means validation has failed
19+
Validate(old, new v1.CustomResourceDefinition) error
20+
// Name returns a human-readable name for the validation
21+
Name() string
22+
}
23+
24+
// ValidateFunc is a function to validate a CustomResourceDefinition
25+
// for safe upgrades. It accepts the old and new CRDs and returns an
26+
// error if performing an upgrade from old -> new is unsafe.
27+
type ValidateFunc func(old, new v1.CustomResourceDefinition) error
28+
29+
// ValidationFunc is a helper to wrap a ValidateFunc
30+
// as an implementation of the Validation interface
31+
type ValidationFunc struct {
32+
name string
33+
validateFunc ValidateFunc
34+
}
35+
36+
func NewValidationFunc(name string, vfunc ValidateFunc) Validation {
37+
return &ValidationFunc{
38+
name: name,
39+
validateFunc: vfunc,
40+
}
41+
}
42+
43+
func (vf *ValidationFunc) Name() string {
44+
return vf.name
45+
}
46+
47+
func (vf *ValidationFunc) Validate(old, new v1.CustomResourceDefinition) error {
48+
return vf.validateFunc(old, new)
49+
}
50+
51+
type Validator struct {
52+
Validations []Validation
53+
}
54+
55+
func (v *Validator) Validate(old, new v1.CustomResourceDefinition) error {
56+
validateErrs := []error{}
57+
for _, validation := range v.Validations {
58+
if err := validation.Validate(old, new); err != nil {
59+
formattedErr := fmt.Errorf("CustomResourceDefinition %s failed upgrade safety validation. %q validation failed: %w",
60+
new.Name, validation.Name(), err)
61+
62+
validateErrs = append(validateErrs, formattedErr)
63+
}
64+
}
65+
if len(validateErrs) > 0 {
66+
return errors.Join(validateErrs...)
67+
}
68+
return nil
69+
}
70+
71+
func NoScopeChange(old, new v1.CustomResourceDefinition) error {
72+
if old.Spec.Scope != new.Spec.Scope {
73+
return fmt.Errorf("scope changed from %q to %q", old.Spec.Scope, new.Spec.Scope)
74+
}
75+
return nil
76+
}
77+
78+
func NoStoredVersionRemoved(old, new v1.CustomResourceDefinition) error {
79+
newVersions := sets.New[string]()
80+
for _, version := range new.Spec.Versions {
81+
if !newVersions.Has(version.Name) {
82+
newVersions.Insert(version.Name)
83+
}
84+
}
85+
86+
for _, storedVersion := range old.Status.StoredVersions {
87+
if !newVersions.Has(storedVersion) {
88+
return fmt.Errorf("stored version %q removed", storedVersion)
89+
}
90+
}
91+
92+
return nil
93+
}

0 commit comments

Comments
 (0)