From 3ba544403ad28eeee17ee06410c674554b0e731a Mon Sep 17 00:00:00 2001 From: Milos Tomic Date: Wed, 24 Jan 2024 20:31:16 +0100 Subject: [PATCH 1/4] aws mock & envtest suite --- components/kcp/Makefile | 2 +- components/kcp/cmd/main.go | 62 +--- components/kcp/go.mod | 2 + .../cloud-control/iprange_controller.go | 29 +- .../cloud-control/nfsinstance_controller.go | 28 +- .../cloud-control/scope_controller.go | 15 + .../controller/cloud-control/suite_test.go | 4 +- .../cloud-control/vpcpeering_controller.go | 9 +- components/kcp/pkg/kcp/scope/state.go | 10 +- .../kcp/pkg/provider/aws/iprange/state.go | 10 +- components/kcp/pkg/provider/aws/mock/ctx.go | 12 + components/kcp/pkg/provider/aws/mock/error.go | 19 ++ .../kcp/pkg/provider/aws/mock/nfsStore.go | 200 +++++++++++ .../kcp/pkg/provider/aws/mock/scopeStore.go | 30 ++ .../kcp/pkg/provider/aws/mock/server.go | 43 +++ components/kcp/pkg/provider/aws/mock/type.go | 44 +++ .../kcp/pkg/provider/aws/mock/vpcStore.go | 144 ++++++++ .../kcp/pkg/provider/aws/nfsinstance/state.go | 10 +- components/kcp/pkg/skr/runtime/alias.go | 4 + components/kcp/pkg/testinfra/crd.go | 150 +++++++++ components/kcp/pkg/testinfra/suite.go | 312 ++++++++++++++++++ 21 files changed, 1074 insertions(+), 65 deletions(-) create mode 100644 components/kcp/pkg/provider/aws/mock/ctx.go create mode 100644 components/kcp/pkg/provider/aws/mock/error.go create mode 100644 components/kcp/pkg/provider/aws/mock/nfsStore.go create mode 100644 components/kcp/pkg/provider/aws/mock/scopeStore.go create mode 100644 components/kcp/pkg/provider/aws/mock/server.go create mode 100644 components/kcp/pkg/provider/aws/mock/type.go create mode 100644 components/kcp/pkg/provider/aws/mock/vpcStore.go create mode 100644 components/kcp/pkg/testinfra/crd.go create mode 100644 components/kcp/pkg/testinfra/suite.go diff --git a/components/kcp/Makefile b/components/kcp/Makefile index 3e9011d2b..ce1bbb1bf 100644 --- a/components/kcp/Makefile +++ b/components/kcp/Makefile @@ -62,7 +62,7 @@ vet: ## Run go vet against code. .PHONY: test test: manifests generate fmt vet envtest ## Run tests. - KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test ./... -coverprofile cover.out + LOCALBIN="$(LOCALBIN)" KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test ./... -coverprofile cover.out GOLANGCI_LINT = $(shell pwd)/bin/golangci-lint GOLANGCI_LINT_VERSION ?= v1.54.2 diff --git a/components/kcp/cmd/main.go b/components/kcp/cmd/main.go index fd8963201..f98533601 100644 --- a/components/kcp/cmd/main.go +++ b/components/kcp/cmd/main.go @@ -19,27 +19,14 @@ package main import ( "flag" "fmt" - kcpscope "github.com/kyma-project/cloud-manager/components/kcp/pkg/kcp/scope" scopeclient "github.com/kyma-project/cloud-manager/components/kcp/pkg/kcp/scope/client" - "os" - - "github.com/kyma-project/cloud-manager/components/kcp/pkg/common/abstractions" - "github.com/kyma-project/cloud-manager/components/kcp/pkg/common/actions/focal" - "github.com/kyma-project/cloud-manager/components/kcp/pkg/iprange" - "github.com/kyma-project/cloud-manager/components/kcp/pkg/nfsinstance" - awsiprange "github.com/kyma-project/cloud-manager/components/kcp/pkg/provider/aws/iprange" awsiprangeclient "github.com/kyma-project/cloud-manager/components/kcp/pkg/provider/aws/iprange/client" - awsnfsinstance "github.com/kyma-project/cloud-manager/components/kcp/pkg/provider/aws/nfsinstance" awsnfsinstanceclient "github.com/kyma-project/cloud-manager/components/kcp/pkg/provider/aws/nfsinstance/client" - azureiprange "github.com/kyma-project/cloud-manager/components/kcp/pkg/provider/azure/iprange" - azurenfsinstance "github.com/kyma-project/cloud-manager/components/kcp/pkg/provider/azure/nfsinstance" - gcpiprange "github.com/kyma-project/cloud-manager/components/kcp/pkg/provider/gcp/iprange" gcpiprangeclient "github.com/kyma-project/cloud-manager/components/kcp/pkg/provider/gcp/iprange/client" - gcpnfsinstance "github.com/kyma-project/cloud-manager/components/kcp/pkg/provider/gcp/nfsinstance" gcpFilestoreClient "github.com/kyma-project/cloud-manager/components/kcp/pkg/provider/gcp/nfsinstance/client" - skrruntime "github.com/kyma-project/cloud-manager/components/kcp/pkg/skr/runtime" - "github.com/kyma-project/cloud-manager/components/lib/composed" + "os" + skrruntime "github.com/kyma-project/cloud-manager/components/kcp/pkg/skr/runtime" "sigs.k8s.io/controller-runtime/pkg/client" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) @@ -71,6 +58,7 @@ func init() { utilruntime.Must(clientgoscheme.AddToScheme(kcpScheme)) utilruntime.Must(cloudcontrolv1beta1.AddToScheme(kcpScheme)) + utilruntime.Must(clientgoscheme.AddToScheme(skrScheme)) utilruntime.Must(cloudresourcesv1beta1.AddToScheme(skrScheme)) //+kubebuilder:scaffold:scheme } @@ -140,44 +128,30 @@ func main() { } // KCP Controllers - if err = (&cloudcontrolcontroller.ScopeReconciler{ - Reconciler: kcpscope.NewScopeReconciler(kcpscope.NewStateFactory( - composed.NewStateFactory(composed.NewStateClusterFromManager(mgr)), - abstractions.NewFileReader(), - scopeclient.NewAwsStsGardenClientProvider(), - )), - }).SetupWithManager(mgr); err != nil { + if err = cloudcontrolcontroller.NewScopeReconciler(mgr, scopeclient.NewAwsStsGardenClientProvider()). + SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "Scope") os.Exit(1) } - if err = (&cloudcontrolcontroller.NfsInstanceReconciler{ - Reconciler: nfsinstance.NewNfsInstanceReconciler( - composed.NewStateFactory(composed.NewStateClusterFromManager(mgr)), - focal.NewStateFactory(), - awsnfsinstance.NewStateFactory(awsnfsinstanceclient.NewClientProvider(), abstractions.NewOSEnvironment()), - azurenfsinstance.NewStateFactory(), - gcpnfsinstance.NewStateFactory(gcpFilestoreClient.NewFilestoreClient(), abstractions.NewOSEnvironment()), - ), - }).SetupWithManager(mgr); err != nil { + if err = cloudcontrolcontroller.NewNfsInstanceReconciler( + mgr, + awsnfsinstanceclient.NewClientProvider(), + gcpFilestoreClient.NewFilestoreClient(), + ). + SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "NfsInstance") os.Exit(1) } - if err = (&cloudcontrolcontroller.VpcPeeringReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - }).SetupWithManager(mgr); err != nil { + if err = cloudcontrolcontroller.NewVpcPeeringReconciler(mgr).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "VpcPeering") os.Exit(1) } - if err = (&cloudcontrolcontroller.IpRangeReconciler{ - Reconciler: iprange.NewIPRangeReconciler( - composed.NewStateFactory(composed.NewStateClusterFromManager(mgr)), - focal.NewStateFactory(), - awsiprange.NewStateFactory(awsiprangeclient.NewClientProvider(), abstractions.NewOSEnvironment()), - azureiprange.NewStateFactory(nil), - gcpiprange.NewStateFactory(gcpiprangeclient.NewServiceNetworkingClient(), gcpiprangeclient.NewComputeClient(), abstractions.NewOSEnvironment()), - ), - }).SetupWithManager(mgr); err != nil { + if err = cloudcontrolcontroller.NewIpRangeReconciler( + mgr, + awsiprangeclient.NewClientProvider(), + gcpiprangeclient.NewServiceNetworkingClient(), + gcpiprangeclient.NewComputeClient(), + ).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "IpRange") os.Exit(1) } diff --git a/components/kcp/go.mod b/components/kcp/go.mod index d1020c639..974e691db 100644 --- a/components/kcp/go.mod +++ b/components/kcp/go.mod @@ -17,6 +17,7 @@ require ( github.com/kyma-project/cloud-manager/components/lib v0.0.0 github.com/onsi/ginkgo/v2 v2.13.0 github.com/onsi/gomega v1.29.0 + github.com/stretchr/testify v1.8.4 google.golang.org/api v0.156.0 k8s.io/api v0.29.0 k8s.io/apimachinery v0.29.0 @@ -73,6 +74,7 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_golang v1.16.0 // indirect github.com/prometheus/client_model v0.4.0 // indirect github.com/prometheus/common v0.44.0 // indirect diff --git a/components/kcp/internal/controller/cloud-control/iprange_controller.go b/components/kcp/internal/controller/cloud-control/iprange_controller.go index 63704b8dd..65c1c2f96 100644 --- a/components/kcp/internal/controller/cloud-control/iprange_controller.go +++ b/components/kcp/internal/controller/cloud-control/iprange_controller.go @@ -19,11 +19,38 @@ package cloudresources import ( "context" cloudresourcesv1beta1 "github.com/kyma-project/cloud-manager/components/kcp/api/cloud-control/v1beta1" + "github.com/kyma-project/cloud-manager/components/kcp/pkg/common/abstractions" + "github.com/kyma-project/cloud-manager/components/kcp/pkg/common/actions/focal" "github.com/kyma-project/cloud-manager/components/kcp/pkg/iprange" + awsclient "github.com/kyma-project/cloud-manager/components/kcp/pkg/provider/aws/client" + awsiprange "github.com/kyma-project/cloud-manager/components/kcp/pkg/provider/aws/iprange" + iprangeclient "github.com/kyma-project/cloud-manager/components/kcp/pkg/provider/aws/iprange/client" + azureiprange "github.com/kyma-project/cloud-manager/components/kcp/pkg/provider/azure/iprange" + gcpclient "github.com/kyma-project/cloud-manager/components/kcp/pkg/provider/gcp/client" + gcpiprange "github.com/kyma-project/cloud-manager/components/kcp/pkg/provider/gcp/iprange" + gcpiprangeclient "github.com/kyma-project/cloud-manager/components/kcp/pkg/provider/gcp/iprange/client" + "github.com/kyma-project/cloud-manager/components/lib/composed" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/manager" ) -// IpRangeReconciler reconciles a IpRange object +func NewIpRangeReconciler( + mgr manager.Manager, + awsProvider awsclient.SkrClientProvider[iprangeclient.Client], + gcpSvcNetProvider gcpclient.ClientProvider[gcpiprangeclient.ServiceNetworkingClient], + gcpComputeProvider gcpclient.ClientProvider[gcpiprangeclient.ComputeClient], +) *IpRangeReconciler { + return &IpRangeReconciler{ + Reconciler: iprange.NewIPRangeReconciler( + composed.NewStateFactory(composed.NewStateClusterFromManager(mgr)), + focal.NewStateFactory(), + awsiprange.NewStateFactory(awsProvider, abstractions.NewOSEnvironment()), + azureiprange.NewStateFactory(nil), + gcpiprange.NewStateFactory(gcpSvcNetProvider, gcpComputeProvider, abstractions.NewOSEnvironment()), + ), + } +} + type IpRangeReconciler struct { Reconciler *iprange.IPRangeReconciler } diff --git a/components/kcp/internal/controller/cloud-control/nfsinstance_controller.go b/components/kcp/internal/controller/cloud-control/nfsinstance_controller.go index 6d79a71bf..4226d2cea 100644 --- a/components/kcp/internal/controller/cloud-control/nfsinstance_controller.go +++ b/components/kcp/internal/controller/cloud-control/nfsinstance_controller.go @@ -19,11 +19,37 @@ package cloudresources import ( "context" cloudresourcesv1beta1 "github.com/kyma-project/cloud-manager/components/kcp/api/cloud-control/v1beta1" + "github.com/kyma-project/cloud-manager/components/kcp/pkg/common/abstractions" + "github.com/kyma-project/cloud-manager/components/kcp/pkg/common/actions/focal" "github.com/kyma-project/cloud-manager/components/kcp/pkg/nfsinstance" + awsclient "github.com/kyma-project/cloud-manager/components/kcp/pkg/provider/aws/client" + awsnfsinstance "github.com/kyma-project/cloud-manager/components/kcp/pkg/provider/aws/nfsinstance" + awsnfsinstanceclient "github.com/kyma-project/cloud-manager/components/kcp/pkg/provider/aws/nfsinstance/client" + azurenfsinstance "github.com/kyma-project/cloud-manager/components/kcp/pkg/provider/azure/nfsinstance" + gcpclient "github.com/kyma-project/cloud-manager/components/kcp/pkg/provider/gcp/client" + gcpnfsinstance "github.com/kyma-project/cloud-manager/components/kcp/pkg/provider/gcp/nfsinstance" + gcpnfsinstanceclient "github.com/kyma-project/cloud-manager/components/kcp/pkg/provider/gcp/nfsinstance/client" + "github.com/kyma-project/cloud-manager/components/lib/composed" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/manager" ) -// NfsInstanceReconciler reconciles a NfsInstance object +func NewNfsInstanceReconciler( + mgr manager.Manager, + awsSkrProvider awsclient.SkrClientProvider[awsnfsinstanceclient.Client], + filestoreClientProvider gcpclient.ClientProvider[gcpnfsinstanceclient.FilestoreClient], +) *NfsInstanceReconciler { + return &NfsInstanceReconciler{ + Reconciler: nfsinstance.NewNfsInstanceReconciler( + composed.NewStateFactory(composed.NewStateClusterFromManager(mgr)), + focal.NewStateFactory(), + awsnfsinstance.NewStateFactory(awsSkrProvider, abstractions.NewOSEnvironment()), + azurenfsinstance.NewStateFactory(), + gcpnfsinstance.NewStateFactory(filestoreClientProvider, abstractions.NewOSEnvironment()), + ), + } +} + type NfsInstanceReconciler struct { Reconciler *nfsinstance.NfsInstanceReconciler } diff --git a/components/kcp/internal/controller/cloud-control/scope_controller.go b/components/kcp/internal/controller/cloud-control/scope_controller.go index 8304cc268..e24d61cd6 100644 --- a/components/kcp/internal/controller/cloud-control/scope_controller.go +++ b/components/kcp/internal/controller/cloud-control/scope_controller.go @@ -3,17 +3,32 @@ package cloudresources import ( "context" cloudcontrolv1beta1 "github.com/kyma-project/cloud-manager/components/kcp/api/cloud-control/v1beta1" + "github.com/kyma-project/cloud-manager/components/kcp/pkg/common/abstractions" kcpscope "github.com/kyma-project/cloud-manager/components/kcp/pkg/kcp/scope" + scopeclient "github.com/kyma-project/cloud-manager/components/kcp/pkg/kcp/scope/client" + awsclient "github.com/kyma-project/cloud-manager/components/kcp/pkg/provider/aws/client" "github.com/kyma-project/cloud-manager/components/kcp/pkg/util" + "github.com/kyma-project/cloud-manager/components/lib/composed" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" ) +func NewScopeReconciler(mgr manager.Manager, awsStsClientProvider awsclient.GardenClientProvider[scopeclient.AwsStsClient]) *ScopeReconciler { + return &ScopeReconciler{ + Reconciler: kcpscope.NewScopeReconciler(kcpscope.NewStateFactory( + composed.NewStateFactory(composed.NewStateClusterFromManager(mgr)), + abstractions.NewFileReader(), + awsStsClientProvider, + )), + } +} + type ScopeReconciler struct { Reconciler kcpscope.ScopeReconciler } diff --git a/components/kcp/internal/controller/cloud-control/suite_test.go b/components/kcp/internal/controller/cloud-control/suite_test.go index 8c8c50d90..6f1674f01 100644 --- a/components/kcp/internal/controller/cloud-control/suite_test.go +++ b/components/kcp/internal/controller/cloud-control/suite_test.go @@ -32,7 +32,7 @@ import ( logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" - cloudresourcesv1beta1 "github.com/kyma-project/cloud-manager/components/kcp/api/cloud-control/v1beta1" + cloudcontrolv1beta1 "github.com/kyma-project/cloud-manager/components/kcp/api/cloud-control/v1beta1" //+kubebuilder:scaffold:imports ) @@ -72,7 +72,7 @@ var _ = BeforeSuite(func() { Expect(err).NotTo(HaveOccurred()) Expect(cfg).NotTo(BeNil()) - err = cloudresourcesv1beta1.AddToScheme(scheme.Scheme) + err = cloudcontrolv1beta1.AddToScheme(scheme.Scheme) Expect(err).NotTo(HaveOccurred()) //+kubebuilder:scaffold:scheme diff --git a/components/kcp/internal/controller/cloud-control/vpcpeering_controller.go b/components/kcp/internal/controller/cloud-control/vpcpeering_controller.go index 046c1b38a..0c67ae46c 100644 --- a/components/kcp/internal/controller/cloud-control/vpcpeering_controller.go +++ b/components/kcp/internal/controller/cloud-control/vpcpeering_controller.go @@ -19,6 +19,7 @@ package cloudresources import ( "context" "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/manager" "k8s.io/apimachinery/pkg/runtime" ctrl "sigs.k8s.io/controller-runtime" @@ -28,7 +29,13 @@ import ( cloudresourcesv1beta1 "github.com/kyma-project/cloud-manager/components/kcp/api/cloud-control/v1beta1" ) -// VpcPeeringReconciler reconciles a VpcPeering object +func NewVpcPeeringReconciler(mgr manager.Manager) *VpcPeeringReconciler { + return &VpcPeeringReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + } +} + type VpcPeeringReconciler struct { client.Client record.EventRecorder diff --git a/components/kcp/pkg/kcp/scope/state.go b/components/kcp/pkg/kcp/scope/state.go index e55314660..f869238be 100644 --- a/components/kcp/pkg/kcp/scope/state.go +++ b/components/kcp/pkg/kcp/scope/state.go @@ -5,7 +5,7 @@ import ( gardenerClient "github.com/gardener/gardener/pkg/client/core/clientset/versioned/typed/core/v1beta1" cloudcontrolv1beta1 "github.com/kyma-project/cloud-manager/components/kcp/api/cloud-control/v1beta1" "github.com/kyma-project/cloud-manager/components/kcp/pkg/common/abstractions" - "github.com/kyma-project/cloud-manager/components/kcp/pkg/kcp/scope/client" + scopeclient "github.com/kyma-project/cloud-manager/components/kcp/pkg/kcp/scope/client" awsClient "github.com/kyma-project/cloud-manager/components/kcp/pkg/provider/aws/client" skrruntime "github.com/kyma-project/cloud-manager/components/kcp/pkg/skr/runtime" "github.com/kyma-project/cloud-manager/components/kcp/pkg/util" @@ -22,7 +22,7 @@ type StateFactory interface { func NewStateFactory( baseStateFactory composed.StateFactory, fileReader abstractions.FileReader, - awsStsClientProvider awsClient.GardenClientProvider[client.AwsStsClient], + awsStsClientProvider awsClient.GardenClientProvider[scopeclient.AwsStsClient], ) StateFactory { return &stateFactory{ baseStateFactory: baseStateFactory, @@ -34,7 +34,7 @@ func NewStateFactory( type stateFactory struct { baseStateFactory composed.StateFactory fileReader abstractions.FileReader - awsStsClientProvider awsClient.GardenClientProvider[client.AwsStsClient] + awsStsClientProvider awsClient.GardenClientProvider[scopeclient.AwsStsClient] } func (f *stateFactory) NewState(req ctrl.Request) *State { @@ -49,7 +49,7 @@ func (f *stateFactory) NewState(req ctrl.Request) *State { // ===================================================================== -func newState(baseState composed.State, fileReader abstractions.FileReader, awsStsClientProvider awsClient.GardenClientProvider[client.AwsStsClient]) *State { +func newState(baseState composed.State, fileReader abstractions.FileReader, awsStsClientProvider awsClient.GardenClientProvider[scopeclient.AwsStsClient]) *State { return &State{ State: baseState, fileReader: fileReader, @@ -77,7 +77,7 @@ type State struct { shoot *gardenerTypes.Shoot credentialData map[string]string - awsStsClientProvider awsClient.GardenClientProvider[client.AwsStsClient] + awsStsClientProvider awsClient.GardenClientProvider[scopeclient.AwsStsClient] } func (s *State) ObjAsScope() *cloudcontrolv1beta1.Scope { diff --git a/components/kcp/pkg/provider/aws/iprange/state.go b/components/kcp/pkg/provider/aws/iprange/state.go index 9689e411e..d747d5281 100644 --- a/components/kcp/pkg/provider/aws/iprange/state.go +++ b/components/kcp/pkg/provider/aws/iprange/state.go @@ -7,7 +7,7 @@ import ( "github.com/kyma-project/cloud-manager/components/kcp/pkg/common/abstractions" "github.com/kyma-project/cloud-manager/components/kcp/pkg/iprange/types" awsclient "github.com/kyma-project/cloud-manager/components/kcp/pkg/provider/aws/client" - "github.com/kyma-project/cloud-manager/components/kcp/pkg/provider/aws/iprange/client" + iprangeclient "github.com/kyma-project/cloud-manager/components/kcp/pkg/provider/aws/iprange/client" ) // State is state of the IpRange reconciliation for AWS @@ -16,7 +16,7 @@ import ( type State struct { types.State - client client.Client + client iprangeclient.Client vpc *ec2Types.Vpc associatedCidrBlock *ec2Types.VpcCidrBlockAssociation @@ -28,7 +28,7 @@ type StateFactory interface { NewState(ctx context.Context, ipRangeState types.State) (*State, error) } -func NewStateFactory(skrProvider awsclient.SkrClientProvider[client.Client], env abstractions.Environment) StateFactory { +func NewStateFactory(skrProvider awsclient.SkrClientProvider[iprangeclient.Client], env abstractions.Environment) StateFactory { return &stateFactory{ skrProvider: skrProvider, env: env, @@ -36,7 +36,7 @@ func NewStateFactory(skrProvider awsclient.SkrClientProvider[client.Client], env } type stateFactory struct { - skrProvider awsclient.SkrClientProvider[client.Client] + skrProvider awsclient.SkrClientProvider[iprangeclient.Client] env abstractions.Environment } @@ -57,7 +57,7 @@ func (f *stateFactory) NewState(ctx context.Context, ipRangeState types.State) ( return newState(ipRangeState, c), nil } -func newState(ipRangeState types.State, c client.Client) *State { +func newState(ipRangeState types.State, c iprangeclient.Client) *State { return &State{ State: ipRangeState, client: c, diff --git a/components/kcp/pkg/provider/aws/mock/ctx.go b/components/kcp/pkg/provider/aws/mock/ctx.go new file mode 100644 index 000000000..99e53a85a --- /dev/null +++ b/components/kcp/pkg/provider/aws/mock/ctx.go @@ -0,0 +1,12 @@ +package mock + +import "context" + +func isContextCanceled(ctx context.Context) bool { + select { + case <-ctx.Done(): + return true + default: + return false + } +} diff --git a/components/kcp/pkg/provider/aws/mock/error.go b/components/kcp/pkg/provider/aws/mock/error.go new file mode 100644 index 000000000..e9b1f95b0 --- /dev/null +++ b/components/kcp/pkg/provider/aws/mock/error.go @@ -0,0 +1,19 @@ +package mock + +import "fmt" + +func newNotATypeError(expectedType string, obj any) error { + return ¬ATypeError{ + obj: obj, + expectedType: expectedType, + } +} + +type notATypeError struct { + obj any + expectedType string +} + +func (e *notATypeError) Error() string { + return fmt.Sprintf("object %T is not an instance of %s", e.obj, e.expectedType) +} diff --git a/components/kcp/pkg/provider/aws/mock/nfsStore.go b/components/kcp/pkg/provider/aws/mock/nfsStore.go new file mode 100644 index 000000000..4ff0d1b35 --- /dev/null +++ b/components/kcp/pkg/provider/aws/mock/nfsStore.go @@ -0,0 +1,200 @@ +package mock + +import ( + "context" + "fmt" + ec2Types "github.com/aws/aws-sdk-go-v2/service/ec2/types" + "github.com/aws/aws-sdk-go-v2/service/efs" + efsTypes "github.com/aws/aws-sdk-go-v2/service/efs/types" + "github.com/elliotchance/pie/v2" + "github.com/google/uuid" + "k8s.io/utils/pointer" +) + +type NfsConfig interface { +} + +type mountTargetItem struct { + desc efsTypes.MountTargetDescription + sg []string +} + +type nfsStore struct { + sg []ec2Types.SecurityGroup + fs []efsTypes.FileSystemDescription + mountTargets map[string][]mountTargetItem +} + +func filterMatchesTags(tags []ec2Types.Tag, filter ec2Types.Filter) bool { + for _, t := range tags { + tagKey := pointer.StringDeref(t.Key, "") + filterName := pointer.StringDeref(filter.Name, "") + if tagKey != filterName { + continue + } + tagValue := pointer.StringDeref(t.Value, "") + for _, fv := range filter.Values { + if tagValue == fv { + return true + } + } + } + return false +} + +func anyFilterMatchTags(tags []ec2Types.Tag, filters []ec2Types.Filter) bool { + for _, f := range filters { + if filterMatchesTags(tags, f) { + return true + } + } + return false +} + +func (s *nfsStore) DescribeSecurityGroups(ctx context.Context, filters []ec2Types.Filter, groupIds []string) ([]ec2Types.SecurityGroup, error) { + if isContextCanceled(ctx) { + return nil, context.Canceled + } + result := append([]ec2Types.SecurityGroup{}, s.sg...) + if groupIds != nil { + result = pie.Filter(result, func(sg ec2Types.SecurityGroup) bool { + return pie.Contains(groupIds, pointer.StringDeref(sg.GroupId, "")) + }) + } + if filters != nil { + result = pie.Filter(result, func(sg ec2Types.SecurityGroup) bool { + return anyFilterMatchTags(sg.Tags, filters) + }) + } + return result, nil +} + +func (s *nfsStore) CreateSecurityGroup(ctx context.Context, vpcId, name string, tags []ec2Types.Tag) (string, error) { + if isContextCanceled(ctx) { + return "", context.Canceled + } + tags = append(tags, ec2Types.Tag{ + Key: pointer.String("vpc-id"), + Value: pointer.String(vpcId), + }) + sg := ec2Types.SecurityGroup{ + Description: pointer.String(name), + GroupId: pointer.String(uuid.NewString()), + GroupName: pointer.String(name), + Tags: tags, + VpcId: pointer.String(vpcId), + } + s.sg = append(s.sg, sg) + return pointer.StringDeref(sg.GroupId, ""), nil +} + +func (s *nfsStore) AuthorizeSecurityGroupIngress(ctx context.Context, groupId string, ipPermissions []ec2Types.IpPermission) error { + if isContextCanceled(ctx) { + return context.Canceled + } + var securityGroup *ec2Types.SecurityGroup + for _, sg := range s.sg { + if pointer.StringDeref(sg.GroupId, "") == groupId { + securityGroup = &sg + break + } + } + if securityGroup == nil { + return fmt.Errorf("security group with id %s does not exist", groupId) + } + securityGroup.IpPermissions = ipPermissions + return nil +} + +func (s *nfsStore) DescribeFileSystems(ctx context.Context) ([]efsTypes.FileSystemDescription, error) { + if isContextCanceled(ctx) { + return nil, context.Canceled + } + return s.fs, nil +} + +func (s *nfsStore) CreateFileSystem(ctx context.Context, performanceMode efsTypes.PerformanceMode, throughputMode efsTypes.ThroughputMode, tags []efsTypes.Tag) (*efs.CreateFileSystemOutput, error) { + if isContextCanceled(ctx) { + return nil, context.Canceled + } + id := uuid.NewString() + fs := efsTypes.FileSystemDescription{ + FileSystemId: pointer.String(id), + LifeCycleState: efsTypes.LifeCycleStateAvailable, + NumberOfMountTargets: 0, + PerformanceMode: performanceMode, + Tags: tags, + Name: pointer.String(id), + ThroughputMode: throughputMode, + } + s.fs = append(s.fs, fs) + + return &efs.CreateFileSystemOutput{ + CreationTime: fs.CreationTime, + CreationToken: fs.CreationToken, + FileSystemId: fs.FileSystemId, + LifeCycleState: fs.LifeCycleState, + NumberOfMountTargets: fs.NumberOfMountTargets, + OwnerId: fs.OwnerId, + PerformanceMode: fs.PerformanceMode, + SizeInBytes: fs.SizeInBytes, + Tags: fs.Tags, + AvailabilityZoneId: fs.AvailabilityZoneId, + AvailabilityZoneName: fs.AvailabilityZoneName, + Encrypted: fs.Encrypted, + FileSystemArn: fs.FileSystemArn, + FileSystemProtection: fs.FileSystemProtection, + KmsKeyId: fs.KmsKeyId, + Name: fs.Name, + ProvisionedThroughputInMibps: fs.ProvisionedThroughputInMibps, + ThroughputMode: fs.ThroughputMode, + }, nil +} + +func (s *nfsStore) DescribeMountTargets(ctx context.Context, fsId string) ([]efsTypes.MountTargetDescription, error) { + if isContextCanceled(ctx) { + return nil, context.Canceled + } + res, ok := s.mountTargets[fsId] + if !ok { + return nil, nil + } + return pie.Map(res, func(i mountTargetItem) efsTypes.MountTargetDescription { + return i.desc + }), nil +} + +func (s *nfsStore) CreateMountTarget(ctx context.Context, fsId, subnetId string, securityGroups []string) (string, error) { + if isContextCanceled(ctx) { + return "", context.Canceled + } + list := s.mountTargets[fsId] + id := uuid.NewString() + item := mountTargetItem{ + desc: efsTypes.MountTargetDescription{ + FileSystemId: pointer.String(fsId), + LifeCycleState: efsTypes.LifeCycleStateAvailable, + MountTargetId: pointer.String(id), + SubnetId: pointer.String(subnetId), + IpAddress: pointer.String("1.2.3.4"), + }, + sg: securityGroups, + } + list = append(list, item) + s.mountTargets[fsId] = list + return id, nil +} + +func (s *nfsStore) DescribeMountTargetSecurityGroups(ctx context.Context, mountTargetId string) ([]string, error) { + if isContextCanceled(ctx) { + return nil, context.Canceled + } + for _, list := range s.mountTargets { + for _, item := range list { + if pointer.StringDeref(item.desc.MountTargetId, "") == mountTargetId { + return item.sg, nil + } + } + } + return nil, fmt.Errorf("mount target with id %s does not exist", mountTargetId) +} diff --git a/components/kcp/pkg/provider/aws/mock/scopeStore.go b/components/kcp/pkg/provider/aws/mock/scopeStore.go new file mode 100644 index 000000000..504b93e00 --- /dev/null +++ b/components/kcp/pkg/provider/aws/mock/scopeStore.go @@ -0,0 +1,30 @@ +package mock + +import ( + "context" + "github.com/aws/aws-sdk-go-v2/service/sts" + "k8s.io/utils/pointer" +) + +type ScopeConfig interface { + SetAccount(string) +} + +type scopeStore struct { + account string +} + +func (s *scopeStore) SetAccount(account string) { + s.account = account +} + +func (s *scopeStore) GetCallerIdentity(ctx context.Context) (*sts.GetCallerIdentityOutput, error) { + if isContextCanceled(ctx) { + return nil, context.Canceled + } + return &sts.GetCallerIdentityOutput{ + Account: pointer.String(s.account), + Arn: nil, + UserId: nil, + }, nil +} diff --git a/components/kcp/pkg/provider/aws/mock/server.go b/components/kcp/pkg/provider/aws/mock/server.go new file mode 100644 index 000000000..a79358672 --- /dev/null +++ b/components/kcp/pkg/provider/aws/mock/server.go @@ -0,0 +1,43 @@ +package mock + +import ( + "context" + scopeclient "github.com/kyma-project/cloud-manager/components/kcp/pkg/kcp/scope/client" + awsclient "github.com/kyma-project/cloud-manager/components/kcp/pkg/provider/aws/client" + iprangeclient "github.com/kyma-project/cloud-manager/components/kcp/pkg/provider/aws/iprange/client" + nfsinstanceclient "github.com/kyma-project/cloud-manager/components/kcp/pkg/provider/aws/nfsinstance/client" +) + +var _ Server = &server{} + +func New() Server { + return &server{ + vpcStore: &vpcStore{}, + nfsStore: &nfsStore{}, + scopeStore: &scopeStore{}, + } +} + +type server struct { + *vpcStore + *nfsStore + *scopeStore +} + +func (s *server) ScopeGardenProvider() awsclient.GardenClientProvider[scopeclient.AwsStsClient] { + return func(ctx context.Context, region, key, secret string) (scopeclient.AwsStsClient, error) { + return s, nil + } +} + +func (s *server) IpRangeSkrProvider() awsclient.SkrClientProvider[iprangeclient.Client] { + return func(ctx context.Context, region, key, secret, role string) (iprangeclient.Client, error) { + return s, nil + } +} + +func (s *server) NfsInstanceSkrProvider() awsclient.SkrClientProvider[nfsinstanceclient.Client] { + return func(ctx context.Context, region, key, secret, role string) (nfsinstanceclient.Client, error) { + return s, nil + } +} diff --git a/components/kcp/pkg/provider/aws/mock/type.go b/components/kcp/pkg/provider/aws/mock/type.go new file mode 100644 index 000000000..72cff89ef --- /dev/null +++ b/components/kcp/pkg/provider/aws/mock/type.go @@ -0,0 +1,44 @@ +package mock + +import ( + scope "github.com/kyma-project/cloud-manager/components/kcp/pkg/kcp/scope/client" + scopeclient "github.com/kyma-project/cloud-manager/components/kcp/pkg/kcp/scope/client" + awsclient "github.com/kyma-project/cloud-manager/components/kcp/pkg/provider/aws/client" + iprangeclient "github.com/kyma-project/cloud-manager/components/kcp/pkg/provider/aws/iprange/client" + nfsinstanceclient "github.com/kyma-project/cloud-manager/components/kcp/pkg/provider/aws/nfsinstance/client" +) + +type IpRangeClient interface { + iprangeclient.Client +} + +type NfsClient interface { + nfsinstanceclient.Client +} + +type ScopeClient interface { + scope.AwsStsClient +} + +type Clients interface { + IpRangeClient + NfsClient + ScopeClient +} + +type Providers interface { + ScopeGardenProvider() awsclient.GardenClientProvider[scopeclient.AwsStsClient] + IpRangeSkrProvider() awsclient.SkrClientProvider[iprangeclient.Client] + NfsInstanceSkrProvider() awsclient.SkrClientProvider[nfsinstanceclient.Client] + //SkrProvider() awsclient.SkrClientProvider[Clients] +} + +type Server interface { + Clients + + Providers + + VpcConfig + NfsConfig + ScopeConfig +} diff --git a/components/kcp/pkg/provider/aws/mock/vpcStore.go b/components/kcp/pkg/provider/aws/mock/vpcStore.go new file mode 100644 index 000000000..bfc72f173 --- /dev/null +++ b/components/kcp/pkg/provider/aws/mock/vpcStore.go @@ -0,0 +1,144 @@ +package mock + +import ( + "context" + "fmt" + ec2Types "github.com/aws/aws-sdk-go-v2/service/ec2/types" + "github.com/elliotchance/pie/v2" + "github.com/google/uuid" + "k8s.io/utils/pointer" +) + +type VpcSubnet struct { + AZ string + Cidr string + Tags []ec2Types.Tag +} + +type VpcConfig interface { + AddVpc(id, cidr string, tags []ec2Types.Tag, subnets []VpcSubnet) +} + +type vpcEntry struct { + vpc ec2Types.Vpc + subnets []ec2Types.Subnet +} + +type vpcStore struct { + items []vpcEntry +} + +func (s *vpcStore) itemByVpcId(vpcId string) (vpcEntry, error) { + idx := pie.FindFirstUsing(s.items, func(e vpcEntry) bool { + return pointer.StringDeref(e.vpc.VpcId, "") == vpcId + }) + if idx == -1 { + return vpcEntry{}, fmt.Errorf("vpc with id %s does not exist", vpcId) + } + return s.items[idx], nil +} + +// Config implementation ======================================= + +func (s *vpcStore) AddVpc(id, cidr string, tags []ec2Types.Tag, subnets []VpcSubnet) { + s.items = append(s.items, vpcEntry{ + vpc: ec2Types.Vpc{ + VpcId: pointer.String(id), + CidrBlock: pointer.String(cidr), + Tags: tags, + }, + subnets: pie.Map(subnets, func(x VpcSubnet) ec2Types.Subnet { + return ec2Types.Subnet{ + AvailabilityZone: pointer.String(x.AZ), + AvailabilityZoneId: pointer.String(x.AZ), + CidrBlock: pointer.String(x.Cidr), + State: ec2Types.SubnetStateAvailable, + SubnetId: pointer.String(uuid.NewString()), + Tags: append(make([]ec2Types.Tag, 0, len(tags)), tags...), + VpcId: pointer.String(id), + } + }), + }) +} + +// Client implementation ======================================== + +func (s *vpcStore) DescribeVpcs(ctx context.Context) ([]ec2Types.Vpc, error) { + if isContextCanceled(ctx) { + return nil, context.Canceled + } + return pie.Map(s.items, func(e vpcEntry) ec2Types.Vpc { + return e.vpc + }), nil +} + +func (s *vpcStore) AssociateVpcCidrBlock(ctx context.Context, vpcId, cidr string) (*ec2Types.VpcCidrBlockAssociation, error) { + if isContextCanceled(ctx) { + return nil, context.Canceled + } + item, err := s.itemByVpcId(vpcId) + if err == nil { + return nil, err + } + a := ec2Types.VpcCidrBlockAssociation{ + AssociationId: pointer.String(uuid.NewString()), + CidrBlock: pointer.String(cidr), + CidrBlockState: &ec2Types.VpcCidrBlockState{ + State: ec2Types.VpcCidrBlockStateCodeAssociated, + StatusMessage: pointer.String("Associated"), + }, + } + item.vpc.CidrBlockAssociationSet = append(item.vpc.CidrBlockAssociationSet, a) + return &a, nil +} + +func (s *vpcStore) DescribeSubnets(ctx context.Context, vpcId string) ([]ec2Types.Subnet, error) { + if isContextCanceled(ctx) { + return nil, context.Canceled + } + item, err := s.itemByVpcId(vpcId) + if err == nil { + return nil, err + } + return item.subnets, nil +} + +func (s *vpcStore) CreateSubnet(ctx context.Context, vpcId, az, cidr string, tags []ec2Types.Tag) (*ec2Types.Subnet, error) { + if isContextCanceled(ctx) { + return nil, context.Canceled + } + item, err := s.itemByVpcId(vpcId) + if err == nil { + return nil, err + } + subnet := ec2Types.Subnet{ + AvailabilityZone: pointer.String(az), + AvailabilityZoneId: pointer.String(az), + CidrBlock: pointer.String(cidr), + State: ec2Types.SubnetStateAvailable, + SubnetId: pointer.String(uuid.NewString()), + Tags: append(make([]ec2Types.Tag, 0, len(tags)), tags...), + VpcId: pointer.String(vpcId), + } + item.subnets = append(item.subnets, subnet) + return &subnet, nil +} + +func (s *vpcStore) DeleteSubnet(ctx context.Context, subnetId string) error { + if isContextCanceled(ctx) { + return context.Canceled + } + for _, item := range s.items { + idx := -1 + for i, subnet := range item.subnets { + if pointer.StringDeref(subnet.SubnetId, "") == subnetId { + idx = i + } + } + if idx > -1 { + item.subnets = pie.Delete(item.subnets, idx) + return nil + } + } + return fmt.Errorf("subnet with id %s", subnetId) +} diff --git a/components/kcp/pkg/provider/aws/nfsinstance/state.go b/components/kcp/pkg/provider/aws/nfsinstance/state.go index 43ead92ba..20f33c0d2 100644 --- a/components/kcp/pkg/provider/aws/nfsinstance/state.go +++ b/components/kcp/pkg/provider/aws/nfsinstance/state.go @@ -8,13 +8,13 @@ import ( "github.com/kyma-project/cloud-manager/components/kcp/pkg/common/abstractions" nfsinstancetypes "github.com/kyma-project/cloud-manager/components/kcp/pkg/nfsinstance/types" awsclient "github.com/kyma-project/cloud-manager/components/kcp/pkg/provider/aws/client" - "github.com/kyma-project/cloud-manager/components/kcp/pkg/provider/aws/nfsinstance/client" + nfsinstanceclient "github.com/kyma-project/cloud-manager/components/kcp/pkg/provider/aws/nfsinstance/client" ) type State struct { nfsinstancetypes.State - awsClient client.Client + awsClient nfsinstanceclient.Client efs *efsTypes.FileSystemDescription mountTargets []efsTypes.MountTargetDescription @@ -27,7 +27,7 @@ type StateFactory interface { NewState(ctx context.Context, nfsInstanceState nfsinstancetypes.State) (*State, error) } -func NewStateFactory(skrProvider awsclient.SkrClientProvider[client.Client], env abstractions.Environment) StateFactory { +func NewStateFactory(skrProvider awsclient.SkrClientProvider[nfsinstanceclient.Client], env abstractions.Environment) StateFactory { return &stateFactory{ skrProvider: skrProvider, env: env, @@ -35,7 +35,7 @@ func NewStateFactory(skrProvider awsclient.SkrClientProvider[client.Client], env } type stateFactory struct { - skrProvider awsclient.SkrClientProvider[client.Client] + skrProvider awsclient.SkrClientProvider[nfsinstanceclient.Client] env abstractions.Environment } @@ -56,7 +56,7 @@ func (f *stateFactory) NewState(ctx context.Context, nfsInstanceState nfsinstanc return newState(nfsInstanceState, c), nil } -func newState(nfsInstanceState nfsinstancetypes.State, c client.Client) *State { +func newState(nfsInstanceState nfsinstancetypes.State, c nfsinstanceclient.Client) *State { return &State{ State: nfsInstanceState, awsClient: c, diff --git a/components/kcp/pkg/skr/runtime/alias.go b/components/kcp/pkg/skr/runtime/alias.go index 2a9469aba..2275e1167 100644 --- a/components/kcp/pkg/skr/runtime/alias.go +++ b/components/kcp/pkg/skr/runtime/alias.go @@ -9,6 +9,10 @@ var NewLooper = looper.New var NewRegistry = registry.New +var NewRunner = looper.NewSkrRunner + type SkrRegistry = registry.SkrRegistry +type SkrRunner = looper.SkrRunner + type ActiveSkrCollection = looper.ActiveSkrCollection diff --git a/components/kcp/pkg/testinfra/crd.go b/components/kcp/pkg/testinfra/crd.go new file mode 100644 index 000000000..b00fda0e0 --- /dev/null +++ b/components/kcp/pkg/testinfra/crd.go @@ -0,0 +1,150 @@ +package testinfra + +import ( + "fmt" + "io" + "os" + "os/exec" + "path" + "path/filepath" + "strings" + "sync" +) + +var crdsInitialized = false + +var ( + dirSkr string + dirKcp string + dirGarden string + + crdMutex sync.Mutex +) + +func initCrds() error { + crdMutex.Lock() + defer crdMutex.Unlock() + + if crdsInitialized { + return nil + } + + localBin := os.Getenv("LOCALBIN") + if len(localBin) == 0 { + localBin = filepath.Join("..", "..", "bin") + } + + dirSkr = path.Join(localBin, "cloud-manager", "skr") + dirKcp = path.Join(localBin, "cloud-manager", "kcp") + dirGarden = path.Join(localBin, "cloud-manager", "garden") + + // recreate destination directories + if err := createDir(dirSkr); err != nil { + return err + } + if err := createDir(dirKcp); err != nil { + return err + } + if err := createDir(dirGarden); err != nil { + return err + } + + // copy CRDs to destination dirs + configCrdBases := filepath.Join("..", "..", "config", "crd", "bases") + list, err := os.ReadDir(configCrdBases) + if err != nil { + return fmt.Errorf("error reading files in config/crd/bases dir: %w", err) + } + + prefixMap := map[string]string{ + "cloud-control.": dirKcp, + "cloud-resources.": dirSkr, + } + for _, x := range list { + if x.IsDir() { + continue + } + for prefix, dir := range prefixMap { + if strings.HasPrefix(x.Name(), prefix) { + err := copyFile( + filepath.Join(configCrdBases, x.Name()), + filepath.Join(dir, x.Name()), + ) + if err != nil { + return err + } + break + } + } + } + + // generate all gardener CRDs + cmd := exec.Command( + filepath.Join(localBin, "controller-gen"), + "crd:allowDangerousTypes=true", + fmt.Sprintf(`paths="%s/pkg/mod/github.com/gardener/gardener@v1.85.0/pkg/apis/core/v1beta1"`, os.Getenv("GOPATH")), + fmt.Sprintf("output:crd:artifacts:config=%s", dirGarden), + ) + err = cmd.Run() + if err != nil { + return err + } + + // delete all garedener CRDs except few we want to keep + gardenFilesToKeep := map[string]struct{}{ + "core.gardener.cloud_shoots.yaml": {}, + "core.gardener.cloud_secretbindings.yaml": {}, + } + list, err = os.ReadDir(dirGarden) + if err != nil { + return err + } + for _, f := range list { + _, keep := gardenFilesToKeep[f.Name()] + if keep { + continue + } + err = os.Remove(filepath.Join(dirGarden, f.Name())) + if err != nil { + return err + } + } + + crdsInitialized = true + return nil +} + +func createDir(dir string) error { + _, err := os.Stat(dir) + if err != nil && !os.IsNotExist(err) { + return err + } + if err == nil { + // dir exists, remove it first, so it gets created empty + err = os.RemoveAll(dir) + if err != nil { + return err + } + } + if err := os.MkdirAll(dir, 0777); err != nil { + return err + } + return nil +} + +func copyFile(src, dest string) error { + s, err := os.Open(src) + if err != nil { + return err + } + defer s.Close() + + d, err := os.Create(dest) + if err != nil { + return err + } + defer d.Close() + + _, err = io.Copy(d, s) + return err +} diff --git a/components/kcp/pkg/testinfra/suite.go b/components/kcp/pkg/testinfra/suite.go new file mode 100644 index 000000000..6de149cf5 --- /dev/null +++ b/components/kcp/pkg/testinfra/suite.go @@ -0,0 +1,312 @@ +package testinfra + +import ( + "context" + "errors" + "fmt" + gardenapi "github.com/gardener/gardener/pkg/apis/core/v1beta1" + "github.com/go-logr/logr" + cloudcontrolv1beta1 "github.com/kyma-project/cloud-manager/components/kcp/api/cloud-control/v1beta1" + cloudresourcesv1beta1 "github.com/kyma-project/cloud-manager/components/kcp/api/cloud-resources/v1beta1" + cloudcontrolcontroller "github.com/kyma-project/cloud-manager/components/kcp/internal/controller/cloud-control" + cloudresourcescontroller "github.com/kyma-project/cloud-manager/components/kcp/internal/controller/cloud-resources" + awsmock "github.com/kyma-project/cloud-manager/components/kcp/pkg/provider/aws/mock" + gcpiprangeclient "github.com/kyma-project/cloud-manager/components/kcp/pkg/provider/gcp/iprange/client" + gcpFilestoreClient "github.com/kyma-project/cloud-manager/components/kcp/pkg/provider/gcp/nfsinstance/client" + skrruntime "github.com/kyma-project/cloud-manager/components/kcp/pkg/skr/runtime" + skrmanager "github.com/kyma-project/cloud-manager/components/kcp/pkg/skr/runtime/manager" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "k8s.io/klog/v2" + "net/http" + "path/filepath" + goruntime "runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sync" + "testing" +) + +type ClusterType string + +const ( + ClusterTypeKcp = ClusterType("kcp") + ClusterTypeSkr = ClusterType("skr") + ClusterTypeGarden = ClusterType("garden") +) + +var onceLogger = sync.Once{} + +var schemeMap map[ClusterType]*runtime.Scheme + +var logger logr.Logger + +func init() { + schemeMap = map[ClusterType]*runtime.Scheme{ + ClusterTypeKcp: runtime.NewScheme(), + ClusterTypeSkr: runtime.NewScheme(), + ClusterTypeGarden: runtime.NewScheme(), + } + // KCP + utilruntime.Must(clientgoscheme.AddToScheme(schemeMap[ClusterTypeKcp])) + utilruntime.Must(cloudcontrolv1beta1.AddToScheme(schemeMap[ClusterTypeKcp])) + // SKR + utilruntime.Must(clientgoscheme.AddToScheme(schemeMap[ClusterTypeSkr])) + utilruntime.Must(cloudresourcesv1beta1.AddToScheme(schemeMap[ClusterTypeSkr])) + // Garden + utilruntime.Must(clientgoscheme.AddToScheme(schemeMap[ClusterTypeGarden])) + utilruntime.Must(gardenapi.AddToScheme(schemeMap[ClusterTypeGarden])) +} + +type Cluster struct { + t *testing.T + Name string + Type ClusterType + KymaRef klog.ObjectRef + Cfg *rest.Config + K8sClient client.Client + TestEnv *envtest.Environment + Manager manager.Manager + Cancel context.CancelFunc + Ctx context.Context + Registry skrruntime.SkrRegistry + Runner skrruntime.SkrRunner + awsMock awsmock.Server +} + +type BaseSuite struct { + suite.Suite + kcpClusterName string + clusters map[string]*Cluster + + AwsMock awsmock.Server +} + +func (s *BaseSuite) CommonSetupSuite() { + s.AwsMock = awsmock.New() + onceLogger.Do(func() { + logger = zap.New(zap.UseDevMode(true)) + ctrl.SetLogger(logger) + }) +} + +func (s *BaseSuite) CommonTearDownSuite() { + for name, cluster := range s.clusters { + if cluster.Cancel != nil { + cluster.Cancel() + } + err := cluster.TestEnv.Stop() + if err != nil { + s.T().Errorf("Error stopping testenv %s: %s", name, err) + } + } +} + +func (s *BaseSuite) Cluster(name string) *Cluster { + return s.clusters[name] +} + +type setupClusterOptions struct { + kymaRef *klog.ObjectRef +} + +type SetupClusterOptionFunc = func(options setupClusterOptions) + +func WithKymaRef(namespace, name string) SetupClusterOptionFunc { + return func(options setupClusterOptions) { + options.kymaRef = &klog.ObjectRef{ + Name: name, + Namespace: namespace, + } + } +} + +func (s *BaseSuite) SetupCluster(name string, clusterType ClusterType, opts ...SetupClusterOptionFunc) (*Cluster, error) { + if err := initCrds(); err != nil { + return nil, err + } + + options := setupClusterOptions{} + for _, o := range opts { + o(options) + } + + if clusterType == ClusterTypeKcp { + if len(s.kcpClusterName) > 0 { + return nil, fmt.Errorf("only one KCP cluster can be created, and %s is already started", s.kcpClusterName) + } + s.kcpClusterName = name + } + + if clusterType == ClusterTypeSkr { + if options.kymaRef == nil { + return nil, errors.New("cluster of the SKR type must have WithKymaRef option") + } + if len(s.kcpClusterName) == 0 { + return nil, errors.New("an SKR cluster can se started only once a KCP cluster is already started") + } + } + + cluster := &Cluster{ + t: s.T(), + Type: clusterType, + Name: name, + awsMock: s.AwsMock, + } + + // TestEnv + crdMap := map[ClusterType]string{ + ClusterTypeKcp: dirKcp, + ClusterTypeSkr: dirSkr, + ClusterTypeGarden: dirGarden, + } + dir, ok := crdMap[clusterType] + if !ok { + return nil, fmt.Errorf("unknown crd dir for cluster type %s", clusterType) + } + cluster.TestEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{dir}, + ErrorIfCRDPathMissing: true, + BinaryAssetsDirectory: filepath.Join("..", "..", "bin", "k8s", + fmt.Sprintf("1.28.3-%s-%s", goruntime.GOOS, goruntime.GOARCH)), + } + + // Cfg + cfg, err := cluster.TestEnv.Start() + if err != nil { + return nil, err + } + if cfg == nil { + return nil, errors.New("testenv started with nil cfg") + } + cluster.Cfg = cfg + + // Client + sch, ok := schemeMap[clusterType] + if !ok { + return nil, fmt.Errorf("unknown scheme for cluster type %s", clusterType) + } + k8sClient, err := client.New(cfg, client.Options{Scheme: sch}) + if err != nil { + return nil, err + } + if k8sClient == nil { + return nil, errors.New("got nil k8sClient") + } + cluster.K8sClient = k8sClient + + // Ctx + ctx, cancel := context.WithCancel(context.TODO()) + cluster.Ctx = ctx + cluster.Cancel = cancel + + // KymaRef + cluster.KymaRef.Name = options.kymaRef.Name + cluster.KymaRef.Namespace = options.kymaRef.Namespace + + // Manager controller-runtime for KCP + if clusterType == ClusterTypeKcp { + mngr, err := ctrl.NewManager(cfg, ctrl.Options{ + Scheme: sch, + }) + if err != nil { + return nil, fmt.Errorf("error creating controller-runtime manager: %w", err) + } + cluster.Manager = mngr + } + + // SkrManager for SKR + if clusterType == ClusterTypeSkr { + mngr, err := skrmanager.New(cfg, sch, cluster.KymaRef, logger) + if err != nil { + return nil, fmt.Errorf("error creating SKR Manager: %w", err) + } + cluster.Manager = mngr + cluster.Registry = skrruntime.NewRegistry(sch) + kcpCluster := s.clusters[s.kcpClusterName] + cluster.Runner = skrruntime.NewRunner(cluster.Registry, kcpCluster.Manager) + } + + return cluster, nil +} + +func (s *BaseSuite) RunClusters() { + for _, c := range s.clusters { + c.Run() + } +} + +func (c *Cluster) Run() { + if c.Manager == nil { + return + } + go func() { + err := c.Manager.Start(c.Ctx) + assert.NoError(c.t, err) + }() +} + +func (c *Cluster) SetupAllControllers(awsMock awsmock.Server) error { + switch c.Type { + case ClusterTypeSkr: + return c.setupAllSkrControllers() + case ClusterTypeKcp: + return c.setupAllKcpControllers() + } + return fmt.Errorf("unable to setup cluster %s of type %s", c.Name, c.Type) +} + +func (c *Cluster) setupAllKcpControllers() (err error) { + if err = cloudcontrolcontroller.NewScopeReconciler( + c.Manager, + c.awsMock.ScopeGardenProvider(), + ).SetupWithManager(c.Manager); err != nil { + return err + } + if err = cloudcontrolcontroller.NewIpRangeReconciler( + c.Manager, + c.awsMock.IpRangeSkrProvider(), + func(ctx context.Context, httpClient *http.Client) (gcpiprangeclient.ServiceNetworkingClient, error) { + return nil, nil + }, + func(ctx context.Context, httpClient *http.Client) (gcpiprangeclient.ComputeClient, error) { + return nil, nil + }, + ).SetupWithManager(c.Manager); err != nil { + return err + } + if err = cloudcontrolcontroller.NewNfsInstanceReconciler( + c.Manager, + c.awsMock.NfsInstanceSkrProvider(), + func(ctx context.Context, httpClient *http.Client) (gcpFilestoreClient.FilestoreClient, error) { + return nil, nil + }, + ).SetupWithManager(c.Manager); err != nil { + return err + } + if err = cloudcontrolcontroller.NewVpcPeeringReconciler(c.Manager).SetupWithManager(c.Manager); err != nil { + return err + } + return +} + +func (c *Cluster) setupAllSkrControllers() (err error) { + if err = cloudresourcescontroller.SetupCloudResourcesReconciler(c.Registry); err != nil { + return err + } + if err = cloudresourcescontroller.SetupIpRangeReconciler(c.Registry); err != nil { + return err + } + if err = cloudresourcescontroller.SetupAwsNfsVolumeReconciler(c.Registry); err != nil { + return err + } + + return +} From 72b0be561fcf32d9e8c926da61b10122fb42cd08 Mon Sep 17 00:00:00 2001 From: Milos Tomic Date: Mon, 29 Jan 2024 11:50:20 +0100 Subject: [PATCH 2/4] test infra --- components/kcp/Makefile | 4 +- components/kcp/cmd/main.go | 16 +- .../core.gardener.cloud_secretbindings.yaml | 129 + .../gardener/core.gardener.cloud_shoots.yaml | 2570 +++++++++++++++++ .../operator.kyma-project.io_kymas.yaml | 806 ++++++ .../cloud-control/iprange_controller.go | 28 +- .../cloud-control/nfsinstance_controller.go | 24 +- .../cloud-control/scope_aws_test.go | 59 + .../cloud-control/scope_controller.go | 27 +- .../controller/cloud-control/suite_test.go | 68 +- .../cloud-control/vpcpeering_controller.go | 8 + components/kcp/pkg/iprange/reconciler.go | 17 +- .../kcp/pkg/kcp/scope/addKymaFinalizer.go | 2 +- .../kcp/pkg/kcp/scope/createGardenerClient.go | 33 +- .../kcp/pkg/kcp/scope/createScopeAws.go | 6 +- .../pkg/kcp/scope/ensureScopeCommonFields.go | 2 +- .../kcp/pkg/kcp/scope/ignoreScopeObj.go | 13 + .../pkg/kcp/scope/loadGardenerCredentials.go | 6 +- components/kcp/pkg/kcp/scope/loadKyma.go | 8 +- components/kcp/pkg/kcp/scope/loadScopeObj.go | 6 +- components/kcp/pkg/kcp/scope/loadShoot.go | 8 +- components/kcp/pkg/kcp/scope/reconciler.go | 17 + components/kcp/pkg/kcp/scope/saveScope.go | 4 +- components/kcp/pkg/kcp/scope/state.go | 19 +- components/kcp/pkg/nfsinstance/reconciler.go | 17 +- .../provider/aws/gardener/infrastructure.go | 12 +- components/kcp/pkg/skr/runtime/alias.go | 2 + .../kcp/pkg/skr/runtime/registry/builder.go | 6 +- .../kcp/pkg/skr/runtime/registry/registry.go | 13 +- components/kcp/pkg/testinfra/crd.go | 145 +- components/kcp/pkg/testinfra/dsl.go | 279 ++ components/kcp/pkg/testinfra/env.go | 87 + components/kcp/pkg/testinfra/infra.go | 72 + .../pkg/testinfra/restConfigToKubeconfig.go | 42 + components/kcp/pkg/testinfra/run.go | 138 + components/kcp/pkg/testinfra/scheme.go | 29 + components/kcp/pkg/testinfra/suite.go | 312 -- components/kcp/pkg/testinfra/types.go | 68 + components/kcp/pkg/util/debugged/doc.go | 6 + components/kcp/pkg/util/debugged/no.go | 6 + components/kcp/pkg/util/debugged/when.go | 8 + components/kcp/pkg/util/debugged/yes.go | 6 + components/kcp/pkg/util/kyma.go | 37 + components/lib/composed/action.go | 8 +- 44 files changed, 4648 insertions(+), 525 deletions(-) create mode 100644 components/kcp/config/crd/gardener/core.gardener.cloud_secretbindings.yaml create mode 100644 components/kcp/config/crd/gardener/core.gardener.cloud_shoots.yaml create mode 100644 components/kcp/config/crd/operator/operator.kyma-project.io_kymas.yaml create mode 100644 components/kcp/internal/controller/cloud-control/scope_aws_test.go create mode 100644 components/kcp/pkg/kcp/scope/ignoreScopeObj.go create mode 100644 components/kcp/pkg/testinfra/dsl.go create mode 100644 components/kcp/pkg/testinfra/env.go create mode 100644 components/kcp/pkg/testinfra/infra.go create mode 100644 components/kcp/pkg/testinfra/restConfigToKubeconfig.go create mode 100644 components/kcp/pkg/testinfra/run.go create mode 100644 components/kcp/pkg/testinfra/scheme.go delete mode 100644 components/kcp/pkg/testinfra/suite.go create mode 100644 components/kcp/pkg/testinfra/types.go create mode 100644 components/kcp/pkg/util/debugged/doc.go create mode 100644 components/kcp/pkg/util/debugged/no.go create mode 100644 components/kcp/pkg/util/debugged/when.go create mode 100644 components/kcp/pkg/util/debugged/yes.go diff --git a/components/kcp/Makefile b/components/kcp/Makefile index ce1bbb1bf..6b7cd7635 100644 --- a/components/kcp/Makefile +++ b/components/kcp/Makefile @@ -62,7 +62,7 @@ vet: ## Run go vet against code. .PHONY: test test: manifests generate fmt vet envtest ## Run tests. - LOCALBIN="$(LOCALBIN)" KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test ./... -coverprofile cover.out + ENVTEST_K8S_VERSION="$(ENVTEST_K8S_VERSION)" PROJECTROOT="$(PROJECTROOT)" KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test ./... -v -coverprofile cover.out GOLANGCI_LINT = $(shell pwd)/bin/golangci-lint GOLANGCI_LINT_VERSION ?= v1.54.2 @@ -143,6 +143,8 @@ undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/confi ##@ Build Dependencies +PROJECTROOT ?= $(shell pwd) + ## Location to install dependencies to LOCALBIN ?= $(shell pwd)/bin $(LOCALBIN): diff --git a/components/kcp/cmd/main.go b/components/kcp/cmd/main.go index 8b8f00510..862f6f5f5 100644 --- a/components/kcp/cmd/main.go +++ b/components/kcp/cmd/main.go @@ -112,6 +112,7 @@ func main() { } skrRegistry := skrruntime.NewRegistry(skrScheme) + skrLoop := skrruntime.NewLooper(mgr, skrScheme, skrRegistry, mgr.GetLogger()) // SKR Controllers if err = cloudresourcescontroller.SetupCloudResourcesReconciler(skrRegistry); err != nil { @@ -132,30 +133,28 @@ func main() { } // KCP Controllers - if err = cloudcontrolcontroller.NewScopeReconciler(mgr, scopeclient.NewAwsStsGardenClientProvider()). - SetupWithManager(mgr); err != nil { + if err = cloudcontrolcontroller.SetupScopeReconciler(mgr, scopeclient.NewAwsStsGardenClientProvider(), skrLoop); err != nil { setupLog.Error(err, "unable to create controller", "controller", "Scope") os.Exit(1) } - if err = cloudcontrolcontroller.NewNfsInstanceReconciler( + if err = cloudcontrolcontroller.SetupNfsInstanceReconciler( mgr, awsnfsinstanceclient.NewClientProvider(), gcpFilestoreClient.NewFilestoreClient(), - ). - SetupWithManager(mgr); err != nil { + ); err != nil { setupLog.Error(err, "unable to create controller", "controller", "NfsInstance") os.Exit(1) } - if err = cloudcontrolcontroller.NewVpcPeeringReconciler(mgr).SetupWithManager(mgr); err != nil { + if err = cloudcontrolcontroller.SetupVpcPeeringReconciler(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "VpcPeering") os.Exit(1) } - if err = cloudcontrolcontroller.NewIpRangeReconciler( + if err = cloudcontrolcontroller.SetupIpRangeReconciler( mgr, awsiprangeclient.NewClientProvider(), gcpiprangeclient.NewServiceNetworkingClient(), gcpiprangeclient.NewComputeClient(), - ).SetupWithManager(mgr); err != nil { + ); err != nil { setupLog.Error(err, "unable to create controller", "controller", "IpRange") os.Exit(1) } @@ -170,7 +169,6 @@ func main() { os.Exit(1) } - skrLoop := skrruntime.NewLooper(mgr, skrScheme, skrRegistry, mgr.GetLogger()) //skrLoop.AddKymaName("dffb0722-a18c-11ee-8c90-0242ac120002") //skrLoop.AddKymaName("134c0a3c-873d-436a-81c3-9b830a27b73a") //skrLoop.AddKymaName("264bb633-80f7-455b-83b2-f86630a57635") diff --git a/components/kcp/config/crd/gardener/core.gardener.cloud_secretbindings.yaml b/components/kcp/config/crd/gardener/core.gardener.cloud_secretbindings.yaml new file mode 100644 index 000000000..2076cf371 --- /dev/null +++ b/components/kcp/config/crd/gardener/core.gardener.cloud_secretbindings.yaml @@ -0,0 +1,129 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.13.0 + name: secretbindings.core.gardener.cloud +spec: + group: core.gardener.cloud + names: + kind: SecretBinding + listKind: SecretBindingList + plural: secretbindings + singular: secretbinding + scope: Namespaced + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: SecretBinding represents a binding to a secret in the same or + another namespace. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + provider: + description: Provider defines the provider type of the SecretBinding. + This field is immutable. + properties: + type: + description: "Type is the type of the provider. \n For backwards compatibility, + the field can contain multiple providers separated by a comma. However + the usage of single SecretBinding (hence Secret) for different cloud + providers is strongly discouraged." + type: string + required: + - type + type: object + quotas: + description: Quotas is a list of references to Quota objects in the same + or another namespace. This field is immutable. + items: + description: "ObjectReference contains enough information to let you + inspect or modify the referred object. --- New uses of this type are + discouraged because of difficulty describing its usage when embedded + in APIs. 1. Ignored fields. It includes many fields which are not + generally honored. For instance, ResourceVersion and FieldPath are + both very rarely valid in actual usage. 2. Invalid usage help. It + is impossible to add specific help for individual usage. In most + embedded usages, there are particular restrictions like, \"must refer + only to types A and B\" or \"UID not honored\" or \"name must be restricted\". + Those cannot be well described when embedded. 3. Inconsistent validation. + \ Because the usages are different, the validation rules are different + by usage, which makes it hard for users to predict what will happen. + 4. The fields are both imprecise and overly precise. Kind is not + a precise mapping to a URL. This can produce ambiguity during interpretation + and require a REST mapping. In most cases, the dependency is on the + group,resource tuple and the version of the actual struct is irrelevant. + 5. We cannot easily change it. Because this type is embedded in many + locations, updates to this type will affect numerous schemas. Don't + make new APIs embed an underspecified API type they do not control. + \n Instead of using this type, create a locally provided and used + type that is well-focused on your reference. For example, ServiceReferences + for admission registration: https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 + ." + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of an + entire object, this string should contain a valid JSON/Go field + access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within + a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" + (container with index 2 in this pod). This syntax is chosen only + to have some well-defined way of referencing a part of an object. + TODO: this design is not final and this field is subject to change + in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference is + made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + x-kubernetes-map-type: atomic + type: array + secretRef: + description: SecretRef is a reference to a secret object in the same or + another namespace. This field is immutable. + properties: + name: + description: name is unique within a namespace to reference a secret + resource. + type: string + namespace: + description: namespace defines the space within which the secret name + must be unique. + type: string + type: object + x-kubernetes-map-type: atomic + required: + - secretRef + type: object + served: true + storage: true diff --git a/components/kcp/config/crd/gardener/core.gardener.cloud_shoots.yaml b/components/kcp/config/crd/gardener/core.gardener.cloud_shoots.yaml new file mode 100644 index 000000000..ebeefaa1b --- /dev/null +++ b/components/kcp/config/crd/gardener/core.gardener.cloud_shoots.yaml @@ -0,0 +1,2570 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.13.0 + name: shoots.core.gardener.cloud +spec: + group: core.gardener.cloud + names: + kind: Shoot + listKind: ShootList + plural: shoots + singular: shoot + scope: Namespaced + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: Shoot represents a Shoot cluster created and managed by Gardener. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: Specification of the Shoot cluster. If the object's deletion + timestamp is set, this field is immutable. + properties: + addons: + description: Addons contains information about enabled/disabled addons + and their configuration. + properties: + kubernetesDashboard: + description: KubernetesDashboard holds configuration settings + for the kubernetes dashboard addon. + properties: + authenticationMode: + description: AuthenticationMode defines the authentication + mode for the kubernetes-dashboard. + type: string + enabled: + description: Enabled indicates whether the addon is enabled + or not. + type: boolean + required: + - enabled + type: object + nginxIngress: + description: NginxIngress holds configuration settings for the + nginx-ingress addon. + properties: + config: + additionalProperties: + type: string + description: Config contains custom configuration for the + nginx-ingress-controller configuration. See https://github.com/kubernetes/ingress-nginx/blob/master/docs/user-guide/nginx-configuration/configmap.md#configuration-options + type: object + enabled: + description: Enabled indicates whether the addon is enabled + or not. + type: boolean + externalTrafficPolicy: + description: ExternalTrafficPolicy controls the `.spec.externalTrafficPolicy` + value of the load balancer `Service` exposing the nginx-ingress. + Defaults to `Cluster`. + type: string + loadBalancerSourceRanges: + description: LoadBalancerSourceRanges is list of allowed IP + sources for NginxIngress + items: + type: string + type: array + required: + - enabled + type: object + type: object + cloudProfileName: + description: CloudProfileName is a name of a CloudProfile object. + This field is immutable. + type: string + controlPlane: + description: ControlPlane contains general settings for the control + plane of the shoot. + properties: + highAvailability: + description: HighAvailability holds the configuration settings + for high availability of the control plane of a shoot. + properties: + failureTolerance: + description: FailureTolerance holds information about failure + tolerance level of a highly available resource. + properties: + type: + description: Type specifies the type of failure that the + highly available resource can tolerate + type: string + required: + - type + type: object + required: + - failureTolerance + type: object + type: object + dns: + description: DNS contains information about the DNS settings of the + Shoot. + properties: + domain: + description: Domain is the external available domain of the Shoot + cluster. This domain will be written into the kubeconfig that + is handed out to end-users. This field is immutable. + type: string + providers: + description: Providers is a list of DNS providers that shall be + enabled for this shoot cluster. Only relevant if not a default + domain is used. + items: + description: DNSProvider contains information about a DNS provider. + properties: + domains: + description: 'Domains contains information about which domains + shall be included/excluded for this provider. Deprecated: + This field is deprecated and will be removed in Gardener + release v1.87.' + properties: + exclude: + description: Exclude is a list of domains that shall + be excluded. + items: + type: string + type: array + include: + description: Include is a list of domains that shall + be included. + items: + type: string + type: array + type: object + primary: + description: Primary indicates that this DNSProvider is + used for shoot related domains. + type: boolean + secretName: + description: SecretName is a name of a secret containing + credentials for the stated domain and the provider. When + not specified, the Gardener will use the cloud provider + credentials referenced by the Shoot and try to find respective + credentials there (primary provider only). Specifying + this field may override this behavior, i.e. forcing the + Gardener to only look into the given secret. + type: string + type: + description: Type is the DNS provider type. + type: string + zones: + description: 'Zones contains information about which hosted + zones shall be included/excluded for this provider. Deprecated: + This field is deprecated and will be removed in Gardener + release v1.87.' + properties: + exclude: + description: Exclude is a list of domains that shall + be excluded. + items: + type: string + type: array + include: + description: Include is a list of domains that shall + be included. + items: + type: string + type: array + type: object + type: object + type: array + type: object + exposureClassName: + description: ExposureClassName is the optional name of an exposure + class to apply a control plane endpoint exposure strategy. This + field is immutable. + type: string + extensions: + description: Extensions contain type and provider information for + Shoot extensions. + items: + description: Extension contains type and provider information for + Shoot extensions. + properties: + disabled: + description: Disabled allows to disable extensions that were + marked as 'globally enabled' by Gardener administrators. + type: boolean + providerConfig: + description: ProviderConfig is the configuration passed to extension + resource. + type: object + x-kubernetes-preserve-unknown-fields: true + type: + description: Type is the type of the extension resource. + type: string + required: + - type + type: object + type: array + hibernation: + description: Hibernation contains information whether the Shoot is + suspended or not. + properties: + enabled: + description: Enabled specifies whether the Shoot needs to be hibernated + or not. If it is true, the Shoot's desired state is to be hibernated. + If it is false or nil, the Shoot's desired state is to be awakened. + type: boolean + schedules: + description: Schedules determine the hibernation schedules. + items: + description: HibernationSchedule determines the hibernation + schedule of a Shoot. A Shoot will be regularly hibernated + at each start time and will be woken up at each end time. + Start or End can be omitted, though at least one of each has + to be specified. + properties: + end: + description: End is a Cron spec at which time a Shoot will + be woken up. + type: string + location: + description: Location is the time location in which both + start and shall be evaluated. + type: string + start: + description: Start is a Cron spec at which time a Shoot + will be hibernated. + type: string + type: object + type: array + type: object + kubernetes: + description: Kubernetes contains the version and configuration settings + of the control plane components. + properties: + allowPrivilegedContainers: + description: AllowPrivilegedContainers indicates whether privileged + containers are allowed in the Shoot. Defaults to true for Kubernetes + versions below v1.25. Unusable for Kubernetes versions v1.25 + and higher. + type: boolean + clusterAutoscaler: + description: ClusterAutoscaler contains the configuration flags + for the Kubernetes cluster autoscaler. + properties: + expander: + description: 'Expander defines the algorithm to use during + scale up (default: least-waste). See: https://github.com/gardener/autoscaler/blob/machine-controller-manager-provider/cluster-autoscaler/FAQ.md#what-are-expanders.' + type: string + ignoreTaints: + description: IgnoreTaints specifies a list of taint keys to + ignore in node templates when considering to scale a node + group. + items: + type: string + type: array + maxEmptyBulkDelete: + description: 'MaxEmptyBulkDelete specifies the maximum number + of empty nodes that can be deleted at the same time (default: + 10).' + format: int32 + type: integer + maxGracefulTerminationSeconds: + description: 'MaxGracefulTerminationSeconds is the number + of seconds CA waits for pod termination when trying to scale + down a node (default: 600).' + format: int32 + type: integer + maxNodeProvisionTime: + description: 'MaxNodeProvisionTime defines how long CA waits + for node to be provisioned (default: 20 mins).' + type: string + newPodScaleUpDelay: + description: NewPodScaleUpDelay specifies how long CA should + ignore newly created pods before they have to be considered + for scale-up. + type: string + scaleDownDelayAfterAdd: + description: 'ScaleDownDelayAfterAdd defines how long after + scale up that scale down evaluation resumes (default: 1 + hour).' + type: string + scaleDownDelayAfterDelete: + description: 'ScaleDownDelayAfterDelete how long after node + deletion that scale down evaluation resumes, defaults to + scanInterval (default: 0 secs).' + type: string + scaleDownDelayAfterFailure: + description: 'ScaleDownDelayAfterFailure how long after scale + down failure that scale down evaluation resumes (default: + 3 mins).' + type: string + scaleDownUnneededTime: + description: 'ScaleDownUnneededTime defines how long a node + should be unneeded before it is eligible for scale down + (default: 30 mins).' + type: string + scaleDownUtilizationThreshold: + description: 'ScaleDownUtilizationThreshold defines the threshold + in fraction (0.0 - 1.0) under which a node is being removed + (default: 0.5).' + type: number + scanInterval: + description: 'ScanInterval how often cluster is reevaluated + for scale up or down (default: 10 secs).' + type: string + type: object + enableStaticTokenKubeconfig: + description: EnableStaticTokenKubeconfig indicates whether static + token kubeconfig secret will be created for the Shoot cluster. + Defaults to true for Shoots with Kubernetes versions < 1.26. + Defaults to false for Shoots with Kubernetes versions >= 1.26. + Starting Kubernetes 1.27 the field will be locked to false. + type: boolean + kubeAPIServer: + description: KubeAPIServer contains configuration settings for + the kube-apiserver. + properties: + admissionPlugins: + description: AdmissionPlugins contains the list of user-defined + admission plugins (additional to those managed by Gardener), + and, if desired, the corresponding configuration. + items: + description: AdmissionPlugin contains information about + a specific admission plugin and its corresponding configuration. + properties: + config: + description: Config is the configuration of the plugin. + type: object + x-kubernetes-preserve-unknown-fields: true + disabled: + description: Disabled specifies whether this plugin + should be disabled. + type: boolean + kubeconfigSecretName: + description: KubeconfigSecretName specifies the name + of a secret containing the kubeconfig for this admission + plugin. + type: string + name: + description: Name is the name of the plugin. + type: string + required: + - name + type: object + type: array + apiAudiences: + description: APIAudiences are the identifiers of the API. + The service account token authenticator will validate that + tokens used against the API are bound to at least one of + these audiences. Defaults to ["kubernetes"]. + items: + type: string + type: array + auditConfig: + description: AuditConfig contains configuration settings for + the audit of the kube-apiserver. + properties: + auditPolicy: + description: AuditPolicy contains configuration settings + for audit policy of the kube-apiserver. + properties: + configMapRef: + description: ConfigMapRef is a reference to a ConfigMap + object in the same namespace, which contains the + audit policy for the kube-apiserver. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object + instead of an entire object, this string should + contain a valid JSON/Go field access statement, + such as desiredState.manifest.containers[2]. + For example, if the object reference is to a + container within a pod, this would take on a + value like: "spec.containers{name}" (where "name" + refers to the name of the container that triggered + the event) or if no container name is specified + "spec.containers[2]" (container with index 2 + in this pod). This syntax is chosen only to + have some well-defined way of referencing a + part of an object. TODO: this design is not + final and this field is subject to change in + the future.' + type: string + kind: + description: 'Kind of the referent. More info: + https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More + info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which + this reference is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + x-kubernetes-map-type: atomic + type: object + type: object + defaultNotReadyTolerationSeconds: + description: DefaultNotReadyTolerationSeconds indicates the + tolerationSeconds of the toleration for notReady:NoExecute + that is added by default to every pod that does not already + have such a toleration (flag `--default-not-ready-toleration-seconds`). + The field has effect only when the `DefaultTolerationSeconds` + admission plugin is enabled. Defaults to 300. + format: int64 + type: integer + defaultUnreachableTolerationSeconds: + description: DefaultUnreachableTolerationSeconds indicates + the tolerationSeconds of the toleration for unreachable:NoExecute + that is added by default to every pod that does not already + have such a toleration (flag `--default-unreachable-toleration-seconds`). + The field has effect only when the `DefaultTolerationSeconds` + admission plugin is enabled. Defaults to 300. + format: int64 + type: integer + enableAnonymousAuthentication: + description: 'EnableAnonymousAuthentication defines whether + anonymous requests to the secure port of the API server + should be allowed (flag `--anonymous-auth`). See: https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/' + type: boolean + eventTTL: + description: EventTTL controls the amount of time to retain + events. Defaults to 1h. + type: string + featureGates: + additionalProperties: + type: boolean + description: FeatureGates contains information about enabled + feature gates. + type: object + logging: + description: Logging contains configuration for the log level + and HTTP access logs. + properties: + httpAccessVerbosity: + description: HTTPAccessVerbosity is the kube-apiserver + access logs level + format: int32 + type: integer + verbosity: + description: Verbosity is the kube-apiserver log verbosity + level Defaults to 2. + format: int32 + type: integer + type: object + oidcConfig: + description: OIDCConfig contains configuration settings for + the OIDC provider. + properties: + caBundle: + description: If set, the OpenID server's certificate will + be verified by one of the authorities in the oidc-ca-file, + otherwise the host's root CA set will be used. + type: string + clientAuthentication: + description: ClientAuthentication can optionally contain + client configuration used for kubeconfig generation. + properties: + extraConfig: + additionalProperties: + type: string + description: Extra configuration added to kubeconfig's + auth-provider. Must not be any of idp-issuer-url, + client-id, client-secret, idp-certificate-authority, + idp-certificate-authority-data, id-token or refresh-token + type: object + secret: + description: The client Secret for the OpenID Connect + client. + type: string + type: object + clientID: + description: The client ID for the OpenID Connect client, + must be set if oidc-issuer-url is set. + type: string + groupsClaim: + description: If provided, the name of a custom OpenID + Connect claim for specifying user groups. The claim + value is expected to be a string or array of strings. + This flag is experimental, please see the authentication + documentation for further details. + type: string + groupsPrefix: + description: If provided, all groups will be prefixed + with this value to prevent conflicts with other authentication + strategies. + type: string + issuerURL: + description: The URL of the OpenID issuer, only HTTPS + scheme will be accepted. If set, it will be used to + verify the OIDC JSON Web Token (JWT). + type: string + requiredClaims: + additionalProperties: + type: string + description: key=value pairs that describes a required + claim in the ID Token. If set, the claim is verified + to be present in the ID Token with a matching value. + type: object + signingAlgs: + description: List of allowed JOSE asymmetric signing algorithms. + JWTs with a 'alg' header value not in this list will + be rejected. Values are defined by RFC 7518 https://tools.ietf.org/html/rfc7518#section-3.1 + items: + type: string + type: array + usernameClaim: + description: The OpenID claim to use as the user name. + Note that claims other than the default ('sub') is not + guaranteed to be unique and immutable. This flag is + experimental, please see the authentication documentation + for further details. (default "sub") + type: string + usernamePrefix: + description: If provided, all usernames will be prefixed + with this value. If not provided, username claims other + than 'email' are prefixed by the issuer URL to avoid + clashes. To skip any prefixing, provide the value '-'. + type: string + type: object + requests: + description: Requests contains configuration for request-specific + settings for the kube-apiserver. + properties: + maxMutatingInflight: + description: MaxMutatingInflight is the maximum number + of mutating requests in flight at a given time. When + the server exceeds this, it rejects requests. + format: int32 + type: integer + maxNonMutatingInflight: + description: MaxNonMutatingInflight is the maximum number + of non-mutating requests in flight at a given time. + When the server exceeds this, it rejects requests. + format: int32 + type: integer + type: object + runtimeConfig: + additionalProperties: + type: boolean + description: RuntimeConfig contains information about enabled + or disabled APIs. + type: object + serviceAccountConfig: + description: ServiceAccountConfig contains configuration settings + for the service account handling of the kube-apiserver. + properties: + acceptedIssuers: + description: AcceptedIssuers is an additional set of issuers + that are used to determine which service account tokens + are accepted. These values are not used to generate + new service account tokens. Only useful when service + account tokens are also issued by another external system + or a change of the current issuer that is used for generating + tokens is being performed. + items: + type: string + type: array + extendTokenExpiration: + description: ExtendTokenExpiration turns on projected + service account expiration extension during token generation, + which helps safe transition from legacy token to bound + service account token feature. If this flag is enabled, + admission injected tokens would be extended up to 1 + year to prevent unexpected failure during transition, + ignoring value of service-account-max-token-expiration. + type: boolean + issuer: + description: Issuer is the identifier of the service account + token issuer. The issuer will assert this identifier + in "iss" claim of issued tokens. This value is used + to generate new service account tokens. This value is + a string or URI. Defaults to URI of the API server. + type: string + maxTokenExpiration: + description: MaxTokenExpiration is the maximum validity + duration of a token created by the service account token + issuer. If an otherwise valid TokenRequest with a validity + duration larger than this value is requested, a token + will be issued with a validity duration of this value. + This field must be within [30d,90d]. + type: string + type: object + watchCacheSizes: + description: WatchCacheSizes contains configuration of the + API server's watch cache sizes. Configuring these flags + might be useful for large-scale Shoot clusters with a lot + of parallel update requests and a lot of watching controllers + (e.g. large ManagedSeed clusters). When the API server's + watch cache's capacity is too small to cope with the amount + of update requests and watchers for a particular resource, + it might happen that controller watches are permanently + stopped with `too old resource version` errors. Starting + from kubernetes v1.19, the API server's watch cache size + is adapted dynamically and setting the watch cache size + flags will have no effect, except when setting it to 0 (which + disables the watch cache). + properties: + default: + description: 'Default configures the default watch cache + size of the kube-apiserver (flag `--default-watch-cache-size`, + defaults to 100). See: https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/' + format: int32 + type: integer + resources: + description: 'Resources configures the watch cache size + of the kube-apiserver per resource (flag `--watch-cache-sizes`). + See: https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/' + items: + description: ResourceWatchCacheSize contains configuration + of the API server's watch cache size for one specific + resource. + properties: + apiGroup: + description: APIGroup is the API group of the resource + for which the watch cache size should be configured. + An unset value is used to specify the legacy core + API (e.g. for `secrets`). + type: string + resource: + description: Resource is the name of the resource + for which the watch cache size should be configured + (in lowercase plural form, e.g. `secrets`). + type: string + size: + description: CacheSize specifies the watch cache + size that should be configured for the specified + resource. + format: int32 + type: integer + required: + - resource + - size + type: object + type: array + type: object + type: object + kubeControllerManager: + description: KubeControllerManager contains configuration settings + for the kube-controller-manager. + properties: + featureGates: + additionalProperties: + type: boolean + description: FeatureGates contains information about enabled + feature gates. + type: object + horizontalPodAutoscaler: + description: HorizontalPodAutoscalerConfig contains horizontal + pod autoscaler configuration settings for the kube-controller-manager. + properties: + cpuInitializationPeriod: + description: The period after which a ready pod transition + is considered to be the first. + type: string + downscaleStabilization: + description: The configurable window at which the controller + will choose the highest recommendation for autoscaling. + type: string + initialReadinessDelay: + description: The configurable period at which the horizontal + pod autoscaler considers a Pod “not yet ready” given + that it’s unready and it has transitioned to unready + during that time. + type: string + syncPeriod: + description: The period for syncing the number of pods + in horizontal pod autoscaler. + type: string + tolerance: + description: The minimum change (from 1.0) in the desired-to-actual + metrics ratio for the horizontal pod autoscaler to consider + scaling. + type: number + type: object + nodeCIDRMaskSize: + description: NodeCIDRMaskSize defines the mask size for node + cidr in cluster (default is 24). This field is immutable. + format: int32 + type: integer + nodeMonitorGracePeriod: + description: NodeMonitorGracePeriod defines the grace period + before an unresponsive node is marked unhealthy. + type: string + podEvictionTimeout: + description: "PodEvictionTimeout defines the grace period + for deleting pods on failed nodes. Defaults to 2m. \n Deprecated: + The corresponding kube-controller-manager flag `--pod-eviction-timeout` + is deprecated in favor of the kube-apiserver flags `--default-not-ready-toleration-seconds` + and `--default-unreachable-toleration-seconds`. The `--pod-eviction-timeout` + flag does not have effect when the taint besed eviction + is enabled. The taint based eviction is beta (enabled by + default) since Kubernetes 1.13 and GA since Kubernetes 1.18. + Hence, instead of setting this field, set the `spec.kubernetes.kubeAPIServer.defaultNotReadyTolerationSeconds` + and `spec.kubernetes.kubeAPIServer.defaultUnreachableTolerationSeconds`." + type: string + type: object + kubeProxy: + description: KubeProxy contains configuration settings for the + kube-proxy. + properties: + enabled: + description: Enabled indicates whether kube-proxy should be + deployed or not. Depending on the networking extensions + switching kube-proxy off might be rejected. Consulting the + respective documentation of the used networking extension + is recommended before using this field. defaults to true + if not specified. + type: boolean + featureGates: + additionalProperties: + type: boolean + description: FeatureGates contains information about enabled + feature gates. + type: object + mode: + description: Mode specifies which proxy mode to use. defaults + to IPTables. + type: string + type: object + kubeScheduler: + description: KubeScheduler contains configuration settings for + the kube-scheduler. + properties: + featureGates: + additionalProperties: + type: boolean + description: FeatureGates contains information about enabled + feature gates. + type: object + kubeMaxPDVols: + description: 'KubeMaxPDVols allows to configure the `KUBE_MAX_PD_VOLS` + environment variable for the kube-scheduler. Please find + more information here: https://kubernetes.io/docs/concepts/storage/storage-limits/#custom-limits + Note that using this field is considered alpha-/experimental-level + and is on your own risk. You should be aware of all the + side-effects and consequences when changing it.' + type: string + profile: + description: Profile configures the scheduling profile for + the cluster. If not specified, the used profile is "balanced" + (provides the default kube-scheduler behavior). + type: string + type: object + kubelet: + description: Kubelet contains configuration settings for the kubelet. + properties: + containerLogMaxFiles: + description: Maximum number of container log files that can + be present for a container. + format: int32 + type: integer + containerLogMaxSize: + anyOf: + - type: integer + - type: string + description: 'A quantity defines the maximum size of the container + log file before it is rotated. For example: "5Mi" or "256Ki". + Default: 100Mi' + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + cpuCFSQuota: + description: CPUCFSQuota allows you to disable/enable CPU + throttling for Pods. + type: boolean + cpuManagerPolicy: + description: 'CPUManagerPolicy allows to set alternative CPU + management policies (default: none).' + type: string + evictionHard: + description: 'EvictionHard describes a set of eviction thresholds + (e.g. memory.available<1Gi) that if met would trigger a + Pod eviction. Default: memory.available: "100Mi/1Gi/5%" + nodefs.available: "5%" nodefs.inodesFree: "5%" imagefs.available: "5%" + imagefs.inodesFree: "5%"' + properties: + imageFSAvailable: + description: ImageFSAvailable is the threshold for the + free disk space in the imagefs filesystem (docker images + and container writable layers). + type: string + imageFSInodesFree: + description: ImageFSInodesFree is the threshold for the + available inodes in the imagefs filesystem. + type: string + memoryAvailable: + description: MemoryAvailable is the threshold for the + free memory on the host server. + type: string + nodeFSAvailable: + description: NodeFSAvailable is the threshold for the + free disk space in the nodefs filesystem (docker volumes, + logs, etc). + type: string + nodeFSInodesFree: + description: NodeFSInodesFree is the threshold for the + available inodes in the nodefs filesystem. + type: string + type: object + evictionMaxPodGracePeriod: + description: 'EvictionMaxPodGracePeriod describes the maximum + allowed grace period (in seconds) to use when terminating + pods in response to a soft eviction threshold being met. + Default: 90' + format: int32 + type: integer + evictionMinimumReclaim: + description: 'EvictionMinimumReclaim configures the amount + of resources below the configured eviction threshold that + the kubelet attempts to reclaim whenever the kubelet observes + resource pressure. Default: 0 for each resource' + properties: + imageFSAvailable: + anyOf: + - type: integer + - type: string + description: ImageFSAvailable is the threshold for the + disk space reclaim in the imagefs filesystem (docker + images and container writable layers). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + imageFSInodesFree: + anyOf: + - type: integer + - type: string + description: ImageFSInodesFree is the threshold for the + inodes reclaim in the imagefs filesystem. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + memoryAvailable: + anyOf: + - type: integer + - type: string + description: MemoryAvailable is the threshold for the + memory reclaim on the host server. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + nodeFSAvailable: + anyOf: + - type: integer + - type: string + description: NodeFSAvailable is the threshold for the + disk space reclaim in the nodefs filesystem (docker + volumes, logs, etc). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + nodeFSInodesFree: + anyOf: + - type: integer + - type: string + description: NodeFSInodesFree is the threshold for the + inodes reclaim in the nodefs filesystem. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + evictionPressureTransitionPeriod: + description: 'EvictionPressureTransitionPeriod is the duration + for which the kubelet has to wait before transitioning out + of an eviction pressure condition. Default: 4m0s' + type: string + evictionSoft: + description: 'EvictionSoft describes a set of eviction thresholds + (e.g. memory.available<1.5Gi) that if met over a corresponding + grace period would trigger a Pod eviction. Default: memory.available: "200Mi/1.5Gi/10%" + nodefs.available: "10%" nodefs.inodesFree: "10%" imagefs.available: "10%" + imagefs.inodesFree: "10%"' + properties: + imageFSAvailable: + description: ImageFSAvailable is the threshold for the + free disk space in the imagefs filesystem (docker images + and container writable layers). + type: string + imageFSInodesFree: + description: ImageFSInodesFree is the threshold for the + available inodes in the imagefs filesystem. + type: string + memoryAvailable: + description: MemoryAvailable is the threshold for the + free memory on the host server. + type: string + nodeFSAvailable: + description: NodeFSAvailable is the threshold for the + free disk space in the nodefs filesystem (docker volumes, + logs, etc). + type: string + nodeFSInodesFree: + description: NodeFSInodesFree is the threshold for the + available inodes in the nodefs filesystem. + type: string + type: object + evictionSoftGracePeriod: + description: 'EvictionSoftGracePeriod describes a set of eviction + grace periods (e.g. memory.available=1m30s) that correspond + to how long a soft eviction threshold must hold before triggering + a Pod eviction. Default: memory.available: 1m30s nodefs.available: 1m30s + nodefs.inodesFree: 1m30s imagefs.available: 1m30s imagefs.inodesFree: + 1m30s' + properties: + imageFSAvailable: + description: ImageFSAvailable is the grace period for + the ImageFSAvailable eviction threshold. + type: string + imageFSInodesFree: + description: ImageFSInodesFree is the grace period for + the ImageFSInodesFree eviction threshold. + type: string + memoryAvailable: + description: MemoryAvailable is the grace period for the + MemoryAvailable eviction threshold. + type: string + nodeFSAvailable: + description: NodeFSAvailable is the grace period for the + NodeFSAvailable eviction threshold. + type: string + nodeFSInodesFree: + description: NodeFSInodesFree is the grace period for + the NodeFSInodesFree eviction threshold. + type: string + type: object + failSwapOn: + description: FailSwapOn makes the Kubelet fail to start if + swap is enabled on the node. (default true). + type: boolean + featureGates: + additionalProperties: + type: boolean + description: FeatureGates contains information about enabled + feature gates. + type: object + imageGCHighThresholdPercent: + description: 'ImageGCHighThresholdPercent describes the percent + of the disk usage which triggers image garbage collection. + Default: 50' + format: int32 + type: integer + imageGCLowThresholdPercent: + description: 'ImageGCLowThresholdPercent describes the percent + of the disk to which garbage collection attempts to free. + Default: 40' + format: int32 + type: integer + imagePullProgressDeadline: + description: 'ImagePullProgressDeadline describes the time + limit under which if no pulling progress is made, the image + pulling will be cancelled. Default: 1m' + type: string + kubeReserved: + description: 'KubeReserved is the configuration for resources + reserved for kubernetes node components (mainly kubelet + and container runtime). When updating these values, be aware + that cgroup resizes may not succeed on active worker nodes. + Look for the NodeAllocatableEnforced event to determine + if the configuration was applied. Default: cpu=80m,memory=1Gi,pid=20k' + properties: + cpu: + anyOf: + - type: integer + - type: string + description: CPU is the reserved cpu. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + ephemeralStorage: + anyOf: + - type: integer + - type: string + description: EphemeralStorage is the reserved ephemeral-storage. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + memory: + anyOf: + - type: integer + - type: string + description: Memory is the reserved memory. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + pid: + anyOf: + - type: integer + - type: string + description: PID is the reserved process-ids. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + maxPods: + description: 'MaxPods is the maximum number of Pods that are + allowed by the Kubelet. Default: 110' + format: int32 + type: integer + memorySwap: + description: MemorySwap configures swap memory available to + container workloads. + properties: + swapBehavior: + description: 'SwapBehavior configures swap memory available + to container workloads. May be one of {"LimitedSwap", + "UnlimitedSwap"} defaults to: LimitedSwap' + type: string + type: object + podPidsLimit: + description: PodPIDsLimit is the maximum number of process + IDs per pod allowed by the kubelet. + format: int64 + type: integer + protectKernelDefaults: + description: ProtectKernelDefaults ensures that the kernel + tunables are equal to the kubelet defaults. Defaults to + true for Kubernetes v1.26 or later. + type: boolean + registryBurst: + description: 'RegistryBurst is the maximum size of bursty + pulls, temporarily allows pulls to burst to this number, + while still not exceeding registryPullQPS. The value must + not be a negative number. Only used if registryPullQPS is + greater than 0. Default: 10' + format: int32 + type: integer + registryPullQPS: + description: 'RegistryPullQPS is the limit of registry pulls + per second. The value must not be a negative number. Setting + it to 0 means no limit. Default: 5' + format: int32 + type: integer + seccompDefault: + description: SeccompDefault enables the use of `RuntimeDefault` + as the default seccomp profile for all workloads. This requires + the corresponding SeccompDefault feature gate to be enabled + as well. This field is only available for Kubernetes v1.25 + or later. + type: boolean + serializeImagePulls: + description: 'SerializeImagePulls describes whether the images + are pulled one at a time. Default: true' + type: boolean + streamingConnectionIdleTimeout: + description: 'StreamingConnectionIdleTimeout is the maximum + time a streaming connection can be idle before the connection + is automatically closed. This field cannot be set lower + than "30s" or greater than "4h". Default: "4h" for Kubernetes + < v1.26. "5m" for Kubernetes >= v1.26.' + type: string + systemReserved: + description: SystemReserved is the configuration for resources + reserved for system processes not managed by kubernetes + (e.g. journald). When updating these values, be aware that + cgroup resizes may not succeed on active worker nodes. Look + for the NodeAllocatableEnforced event to determine if the + configuration was applied. + properties: + cpu: + anyOf: + - type: integer + - type: string + description: CPU is the reserved cpu. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + ephemeralStorage: + anyOf: + - type: integer + - type: string + description: EphemeralStorage is the reserved ephemeral-storage. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + memory: + anyOf: + - type: integer + - type: string + description: Memory is the reserved memory. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + pid: + anyOf: + - type: integer + - type: string + description: PID is the reserved process-ids. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + version: + description: Version is the semantic Kubernetes version to use + for the Shoot cluster. Defaults to the highest supported minor + and patch version given in the referenced cloud profile. The + version can be omitted completely or partially specified, e.g. + `.`. + type: string + verticalPodAutoscaler: + description: VerticalPodAutoscaler contains the configuration + flags for the Kubernetes vertical pod autoscaler. + properties: + enabled: + description: Enabled specifies whether the Kubernetes VPA + shall be enabled for the shoot cluster. + type: boolean + evictAfterOOMThreshold: + description: 'EvictAfterOOMThreshold defines the threshold + that will lead to pod eviction in case it OOMed in less + than the given threshold since its start and if it has only + one container (default: 10m0s).' + type: string + evictionRateBurst: + description: 'EvictionRateBurst defines the burst of pods + that can be evicted (default: 1)' + format: int32 + type: integer + evictionRateLimit: + description: 'EvictionRateLimit defines the number of pods + that can be evicted per second. A rate limit set to 0 or + -1 will disable the rate limiter (default: -1).' + type: number + evictionTolerance: + description: 'EvictionTolerance defines the fraction of replica + count that can be evicted for update in case more than one + pod can be evicted (default: 0.5).' + type: number + recommendationMarginFraction: + description: 'RecommendationMarginFraction is the fraction + of usage added as the safety margin to the recommended request + (default: 0.15).' + type: number + recommenderInterval: + description: 'RecommenderInterval is the interval how often + metrics should be fetched (default: 1m0s).' + type: string + updaterInterval: + description: 'UpdaterInterval is the interval how often the + updater should run (default: 1m0s).' + type: string + required: + - enabled + type: object + type: object + maintenance: + description: Maintenance contains information about the time window + for maintenance operations and which operations should be performed. + properties: + autoUpdate: + description: AutoUpdate contains information about which constraints + should be automatically updated. + properties: + kubernetesVersion: + description: 'KubernetesVersion indicates whether the patch + Kubernetes version may be automatically updated (default: + true).' + type: boolean + machineImageVersion: + description: 'MachineImageVersion indicates whether the machine + image version may be automatically updated (default: true).' + type: boolean + required: + - kubernetesVersion + type: object + confineSpecUpdateRollout: + description: ConfineSpecUpdateRollout prevents that changes/updates + to the shoot specification will be rolled out immediately. Instead, + they are rolled out during the shoot's maintenance time window. + There is one exception that will trigger an immediate roll out + which is changes to the Spec.Hibernation.Enabled field. + type: boolean + timeWindow: + description: TimeWindow contains information about the time window + for maintenance operations. + properties: + begin: + description: Begin is the beginning of the time window in + the format HHMMSS+ZONE, e.g. "220000+0100". If not present, + a random value will be computed. + pattern: ([0-1][0-9]|2[0-3])[0-5][0-9][0-5][0-9]\+[0-1][0-4]00 + type: string + end: + description: End is the end of the time window in the format + HHMMSS+ZONE, e.g. "220000+0100". If not present, the value + will be computed based on the "Begin" value. + pattern: ([0-1][0-9]|2[0-3])[0-5][0-9][0-5][0-9]\+[0-1][0-4]00 + type: string + required: + - begin + - end + type: object + type: object + monitoring: + description: Monitoring contains information about custom monitoring + configurations for the shoot. + properties: + alerting: + description: Alerting contains information about the alerting + configuration for the shoot cluster. + properties: + emailReceivers: + description: MonitoringEmailReceivers is a list of recipients + for alerts + items: + type: string + type: array + type: object + type: object + networking: + description: Networking contains information about cluster networking + such as CNI Plugin type, CIDRs, ...etc. + properties: + ipFamilies: + description: IPFamilies specifies the IP protocol versions to + use for shoot networking. This field is immutable. See https://github.com/gardener/gardener/blob/master/docs/usage/ipv6.md. + Defaults to ["IPv4"]. + items: + description: IPFamily is a type for specifying an IP protocol + version to use in Gardener clusters. + type: string + type: array + nodes: + description: Nodes is the CIDR of the entire node network. This + field is immutable if the feature gate MutableShootSpecNetworkingNodes + is disabled. + type: string + pods: + description: Pods is the CIDR of the pod network. This field is + immutable. + type: string + providerConfig: + description: ProviderConfig is the configuration passed to network + resource. + type: object + x-kubernetes-preserve-unknown-fields: true + services: + description: Services is the CIDR of the service network. This + field is immutable. + type: string + type: + description: Type identifies the type of the networking plugin. + This field is immutable. + type: string + type: object + provider: + description: Provider contains all provider-specific and provider-relevant + information. + properties: + controlPlaneConfig: + description: ControlPlaneConfig contains the provider-specific + control plane config blob. Please look up the concrete definition + in the documentation of your provider extension. + type: object + x-kubernetes-preserve-unknown-fields: true + infrastructureConfig: + description: InfrastructureConfig contains the provider-specific + infrastructure config blob. Please look up the concrete definition + in the documentation of your provider extension. + type: object + x-kubernetes-preserve-unknown-fields: true + type: + description: Type is the type of the provider. This field is immutable. + type: string + workers: + description: Workers is a list of worker groups. + items: + description: Worker is the base definition of a worker group. + properties: + annotations: + additionalProperties: + type: string + description: Annotations is a map of key/value pairs for + annotations for all the `Node` objects in this worker + pool. + type: object + caBundle: + description: CABundle is a certificate bundle which will + be installed onto every machine of this worker pool. + type: string + cri: + description: CRI contains configurations of CRI support + of every machine in the worker pool. Defaults to a CRI + with name `containerd`. + properties: + containerRuntimes: + description: ContainerRuntimes is the list of the required + container runtimes supported for a worker pool. + items: + description: ContainerRuntime contains information + about worker's available container runtime + properties: + providerConfig: + description: ProviderConfig is the configuration + passed to container runtime resource. + type: object + x-kubernetes-preserve-unknown-fields: true + type: + description: Type is the type of the Container + Runtime. + type: string + required: + - type + type: object + type: array + name: + description: The name of the CRI library. Supported + values are `containerd`. + type: string + required: + - name + type: object + dataVolumes: + description: DataVolumes contains a list of additional worker + volumes. + items: + description: DataVolume contains information about a data + volume. + properties: + encrypted: + description: Encrypted determines if the volume should + be encrypted. + type: boolean + name: + description: Name of the volume to make it referencable. + type: string + size: + description: VolumeSize is the size of the volume. + type: string + type: + description: Type is the type of the volume. + type: string + required: + - name + - size + type: object + type: array + kubeletDataVolumeName: + description: KubeletDataVolumeName contains the name of + a dataVolume that should be used for storing kubelet state. + type: string + kubernetes: + description: Kubernetes contains configuration for Kubernetes + components related to this worker pool. + properties: + kubelet: + description: Kubelet contains configuration settings + for all kubelets of this worker pool. If set, all + `spec.kubernetes.kubelet` settings will be overwritten + for this worker pool (no merge of settings). + properties: + containerLogMaxFiles: + description: Maximum number of container log files + that can be present for a container. + format: int32 + type: integer + containerLogMaxSize: + anyOf: + - type: integer + - type: string + description: 'A quantity defines the maximum size + of the container log file before it is rotated. + For example: "5Mi" or "256Ki". Default: 100Mi' + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + cpuCFSQuota: + description: CPUCFSQuota allows you to disable/enable + CPU throttling for Pods. + type: boolean + cpuManagerPolicy: + description: 'CPUManagerPolicy allows to set alternative + CPU management policies (default: none).' + type: string + evictionHard: + description: 'EvictionHard describes a set of eviction + thresholds (e.g. memory.available<1Gi) that if + met would trigger a Pod eviction. Default: memory.available: "100Mi/1Gi/5%" + nodefs.available: "5%" nodefs.inodesFree: "5%" + imagefs.available: "5%" imagefs.inodesFree: "5%"' + properties: + imageFSAvailable: + description: ImageFSAvailable is the threshold + for the free disk space in the imagefs filesystem + (docker images and container writable layers). + type: string + imageFSInodesFree: + description: ImageFSInodesFree is the threshold + for the available inodes in the imagefs filesystem. + type: string + memoryAvailable: + description: MemoryAvailable is the threshold + for the free memory on the host server. + type: string + nodeFSAvailable: + description: NodeFSAvailable is the threshold + for the free disk space in the nodefs filesystem + (docker volumes, logs, etc). + type: string + nodeFSInodesFree: + description: NodeFSInodesFree is the threshold + for the available inodes in the nodefs filesystem. + type: string + type: object + evictionMaxPodGracePeriod: + description: 'EvictionMaxPodGracePeriod describes + the maximum allowed grace period (in seconds) + to use when terminating pods in response to a + soft eviction threshold being met. Default: 90' + format: int32 + type: integer + evictionMinimumReclaim: + description: 'EvictionMinimumReclaim configures + the amount of resources below the configured eviction + threshold that the kubelet attempts to reclaim + whenever the kubelet observes resource pressure. + Default: 0 for each resource' + properties: + imageFSAvailable: + anyOf: + - type: integer + - type: string + description: ImageFSAvailable is the threshold + for the disk space reclaim in the imagefs + filesystem (docker images and container writable + layers). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + imageFSInodesFree: + anyOf: + - type: integer + - type: string + description: ImageFSInodesFree is the threshold + for the inodes reclaim in the imagefs filesystem. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + memoryAvailable: + anyOf: + - type: integer + - type: string + description: MemoryAvailable is the threshold + for the memory reclaim on the host server. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + nodeFSAvailable: + anyOf: + - type: integer + - type: string + description: NodeFSAvailable is the threshold + for the disk space reclaim in the nodefs filesystem + (docker volumes, logs, etc). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + nodeFSInodesFree: + anyOf: + - type: integer + - type: string + description: NodeFSInodesFree is the threshold + for the inodes reclaim in the nodefs filesystem. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + evictionPressureTransitionPeriod: + description: 'EvictionPressureTransitionPeriod is + the duration for which the kubelet has to wait + before transitioning out of an eviction pressure + condition. Default: 4m0s' + type: string + evictionSoft: + description: 'EvictionSoft describes a set of eviction + thresholds (e.g. memory.available<1.5Gi) that + if met over a corresponding grace period would + trigger a Pod eviction. Default: memory.available: "200Mi/1.5Gi/10%" + nodefs.available: "10%" nodefs.inodesFree: "10%" + imagefs.available: "10%" imagefs.inodesFree: + "10%"' + properties: + imageFSAvailable: + description: ImageFSAvailable is the threshold + for the free disk space in the imagefs filesystem + (docker images and container writable layers). + type: string + imageFSInodesFree: + description: ImageFSInodesFree is the threshold + for the available inodes in the imagefs filesystem. + type: string + memoryAvailable: + description: MemoryAvailable is the threshold + for the free memory on the host server. + type: string + nodeFSAvailable: + description: NodeFSAvailable is the threshold + for the free disk space in the nodefs filesystem + (docker volumes, logs, etc). + type: string + nodeFSInodesFree: + description: NodeFSInodesFree is the threshold + for the available inodes in the nodefs filesystem. + type: string + type: object + evictionSoftGracePeriod: + description: 'EvictionSoftGracePeriod describes + a set of eviction grace periods (e.g. memory.available=1m30s) + that correspond to how long a soft eviction threshold + must hold before triggering a Pod eviction. Default: + memory.available: 1m30s nodefs.available: 1m30s + nodefs.inodesFree: 1m30s imagefs.available: 1m30s + imagefs.inodesFree: 1m30s' + properties: + imageFSAvailable: + description: ImageFSAvailable is the grace period + for the ImageFSAvailable eviction threshold. + type: string + imageFSInodesFree: + description: ImageFSInodesFree is the grace + period for the ImageFSInodesFree eviction + threshold. + type: string + memoryAvailable: + description: MemoryAvailable is the grace period + for the MemoryAvailable eviction threshold. + type: string + nodeFSAvailable: + description: NodeFSAvailable is the grace period + for the NodeFSAvailable eviction threshold. + type: string + nodeFSInodesFree: + description: NodeFSInodesFree is the grace period + for the NodeFSInodesFree eviction threshold. + type: string + type: object + failSwapOn: + description: FailSwapOn makes the Kubelet fail to + start if swap is enabled on the node. (default + true). + type: boolean + featureGates: + additionalProperties: + type: boolean + description: FeatureGates contains information about + enabled feature gates. + type: object + imageGCHighThresholdPercent: + description: 'ImageGCHighThresholdPercent describes + the percent of the disk usage which triggers image + garbage collection. Default: 50' + format: int32 + type: integer + imageGCLowThresholdPercent: + description: 'ImageGCLowThresholdPercent describes + the percent of the disk to which garbage collection + attempts to free. Default: 40' + format: int32 + type: integer + imagePullProgressDeadline: + description: 'ImagePullProgressDeadline describes + the time limit under which if no pulling progress + is made, the image pulling will be cancelled. + Default: 1m' + type: string + kubeReserved: + description: 'KubeReserved is the configuration + for resources reserved for kubernetes node components + (mainly kubelet and container runtime). When updating + these values, be aware that cgroup resizes may + not succeed on active worker nodes. Look for the + NodeAllocatableEnforced event to determine if + the configuration was applied. Default: cpu=80m,memory=1Gi,pid=20k' + properties: + cpu: + anyOf: + - type: integer + - type: string + description: CPU is the reserved cpu. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + ephemeralStorage: + anyOf: + - type: integer + - type: string + description: EphemeralStorage is the reserved + ephemeral-storage. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + memory: + anyOf: + - type: integer + - type: string + description: Memory is the reserved memory. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + pid: + anyOf: + - type: integer + - type: string + description: PID is the reserved process-ids. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + maxPods: + description: 'MaxPods is the maximum number of Pods + that are allowed by the Kubelet. Default: 110' + format: int32 + type: integer + memorySwap: + description: MemorySwap configures swap memory available + to container workloads. + properties: + swapBehavior: + description: 'SwapBehavior configures swap memory + available to container workloads. May be one + of {"LimitedSwap", "UnlimitedSwap"} defaults + to: LimitedSwap' + type: string + type: object + podPidsLimit: + description: PodPIDsLimit is the maximum number + of process IDs per pod allowed by the kubelet. + format: int64 + type: integer + protectKernelDefaults: + description: ProtectKernelDefaults ensures that + the kernel tunables are equal to the kubelet defaults. + Defaults to true for Kubernetes v1.26 or later. + type: boolean + registryBurst: + description: 'RegistryBurst is the maximum size + of bursty pulls, temporarily allows pulls to burst + to this number, while still not exceeding registryPullQPS. + The value must not be a negative number. Only + used if registryPullQPS is greater than 0. Default: + 10' + format: int32 + type: integer + registryPullQPS: + description: 'RegistryPullQPS is the limit of registry + pulls per second. The value must not be a negative + number. Setting it to 0 means no limit. Default: + 5' + format: int32 + type: integer + seccompDefault: + description: SeccompDefault enables the use of `RuntimeDefault` + as the default seccomp profile for all workloads. + This requires the corresponding SeccompDefault + feature gate to be enabled as well. This field + is only available for Kubernetes v1.25 or later. + type: boolean + serializeImagePulls: + description: 'SerializeImagePulls describes whether + the images are pulled one at a time. Default: + true' + type: boolean + streamingConnectionIdleTimeout: + description: 'StreamingConnectionIdleTimeout is + the maximum time a streaming connection can be + idle before the connection is automatically closed. + This field cannot be set lower than "30s" or greater + than "4h". Default: "4h" for Kubernetes < v1.26. + "5m" for Kubernetes >= v1.26.' + type: string + systemReserved: + description: SystemReserved is the configuration + for resources reserved for system processes not + managed by kubernetes (e.g. journald). When updating + these values, be aware that cgroup resizes may + not succeed on active worker nodes. Look for the + NodeAllocatableEnforced event to determine if + the configuration was applied. + properties: + cpu: + anyOf: + - type: integer + - type: string + description: CPU is the reserved cpu. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + ephemeralStorage: + anyOf: + - type: integer + - type: string + description: EphemeralStorage is the reserved + ephemeral-storage. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + memory: + anyOf: + - type: integer + - type: string + description: Memory is the reserved memory. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + pid: + anyOf: + - type: integer + - type: string + description: PID is the reserved process-ids. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + version: + description: Version is the semantic Kubernetes version + to use for the Kubelet in this Worker Group. If not + specified the kubelet version is derived from the + global shoot cluster kubernetes version. version must + be equal or lower than the version of the shoot kubernetes + version. Only one minor version difference to other + worker groups and global kubernetes version is allowed. + type: string + type: object + labels: + additionalProperties: + type: string + description: Labels is a map of key/value pairs for labels + for all the `Node` objects in this worker pool. + type: object + machine: + description: Machine contains information about the machine + type and image. + properties: + architecture: + description: Architecture is CPU architecture of machines + in this worker pool. + type: string + image: + description: Image holds information about the machine + image to use for all nodes of this pool. It will default + to the latest version of the first image stated in + the referenced CloudProfile if no value has been provided. + properties: + name: + description: Name is the name of the image. + type: string + providerConfig: + description: ProviderConfig is the shoot's individual + configuration passed to an extension resource. + type: object + x-kubernetes-preserve-unknown-fields: true + version: + description: Version is the version of the shoot's + image. If version is not provided, it will be + defaulted to the latest version from the CloudProfile. + type: string + required: + - name + type: object + type: + description: Type is the machine type of the worker + group. + type: string + required: + - type + type: object + machineControllerManager: + description: MachineControllerManagerSettings contains configurations + for different worker-pools. Eg. MachineDrainTimeout, MachineHealthTimeout. + properties: + machineCreationTimeout: + description: MachineCreationTimeout is the period after + which creation of the machine is declared failed. + type: string + machineDrainTimeout: + description: MachineDrainTimeout is the period after + which machine is forcefully deleted. + type: string + machineHealthTimeout: + description: MachineHealthTimeout is the period after + which machine is declared failed. + type: string + maxEvictRetries: + description: MaxEvictRetries are the number of eviction + retries on a pod after which drain is declared failed, + and forceful deletion is triggered. + format: int32 + type: integer + nodeConditions: + description: NodeConditions are the set of conditions + if set to true for the period of MachineHealthTimeout, + machine will be declared failed. + items: + type: string + type: array + type: object + maxSurge: + anyOf: + - type: integer + - type: string + description: MaxSurge is maximum number of machines that + are created during an update. This value is divided by + the number of configured zones for a fair distribution. + x-kubernetes-int-or-string: true + maxUnavailable: + anyOf: + - type: integer + - type: string + description: MaxUnavailable is the maximum number of machines + that can be unavailable during an update. This value is + divided by the number of configured zones for a fair distribution. + x-kubernetes-int-or-string: true + maximum: + description: Maximum is the maximum number of machines to + create. This value is divided by the number of configured + zones for a fair distribution. + format: int32 + type: integer + minimum: + description: Minimum is the minimum number of machines to + create. This value is divided by the number of configured + zones for a fair distribution. + format: int32 + type: integer + name: + description: Name is the name of the worker group. + type: string + providerConfig: + description: ProviderConfig is the provider-specific configuration + for this worker pool. + type: object + x-kubernetes-preserve-unknown-fields: true + sysctls: + additionalProperties: + type: string + description: Sysctls is a map of kernel settings to apply + on all machines in this worker pool. + type: object + systemComponents: + description: SystemComponents contains configuration for + system components related to this worker pool + properties: + allow: + description: Allow determines whether the pool should + be allowed to host system components or not (defaults + to true) + type: boolean + required: + - allow + type: object + taints: + description: Taints is a list of taints for all the `Node` + objects in this worker pool. + items: + description: The node this Taint is attached to has the + "effect" on any pod that does not tolerate the Taint. + properties: + effect: + description: Required. The effect of the taint on + pods that do not tolerate the taint. Valid effects + are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: Required. The taint key to be applied + to a node. + type: string + timeAdded: + description: TimeAdded represents the time at which + the taint was added. It is only written for NoExecute + taints. + format: date-time + type: string + value: + description: The taint value corresponding to the + taint key. + type: string + required: + - effect + - key + type: object + type: array + volume: + description: Volume contains information about the volume + type and size. + properties: + encrypted: + description: Encrypted determines if the volume should + be encrypted. + type: boolean + name: + description: Name of the volume to make it referencable. + type: string + size: + description: VolumeSize is the size of the volume. + type: string + type: + description: Type is the type of the volume. + type: string + required: + - size + type: object + zones: + description: Zones is a list of availability zones that + are used to evenly distribute this worker pool. Optional + as not every provider may support availability zones. + items: + type: string + type: array + required: + - machine + - maximum + - minimum + - name + type: object + type: array + workersSettings: + description: WorkersSettings contains settings for all workers. + properties: + sshAccess: + description: SSHAccess contains settings regarding ssh access + to the worker nodes. + properties: + enabled: + description: Enabled indicates whether the SSH access + to the worker nodes is ensured to be enabled or disabled + in systemd. Defaults to true. + type: boolean + required: + - enabled + type: object + type: object + required: + - type + type: object + purpose: + description: Purpose is the purpose class for this cluster. + type: string + region: + description: Region is a name of a region. This field is immutable. + type: string + resources: + description: Resources holds a list of named resource references that + can be referred to in extension configs by their names. + items: + description: NamedResourceReference is a named reference to a resource. + properties: + name: + description: Name of the resource reference. + type: string + resourceRef: + description: ResourceRef is a reference to a resource. + properties: + apiVersion: + description: apiVersion is the API version of the referent + type: string + kind: + description: 'kind is the kind of the referent; More info: + https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'name is the name of the referent; More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + required: + - name + - resourceRef + type: object + type: array + schedulerName: + description: SchedulerName is the name of the responsible scheduler + which schedules the shoot. If not specified, the default scheduler + takes over. This field is immutable. + type: string + secretBindingName: + description: SecretBindingName is the name of the a SecretBinding + that has a reference to the provider secret. The credentials inside + the provider secret will be used to create the shoot in the respective + account. This field is immutable. + type: string + seedName: + description: SeedName is the name of the seed cluster that runs the + control plane of the Shoot. + type: string + seedSelector: + description: SeedSelector is an optional selector which must match + a seed's labels for the shoot to be scheduled on that seed. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that + contains values, a key, and an operator that relates the key + and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship to + a set of values. Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of string values. If the + operator is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator + is "In", and the values array contains only "value". The requirements + are ANDed. + type: object + providerTypes: + description: Providers is optional and can be used by restricting + seeds by their provider type. '*' can be used to enable seeds + regardless of their provider type. + items: + type: string + type: array + type: object + x-kubernetes-map-type: atomic + systemComponents: + description: SystemComponents contains the settings of system components + in the control or data plane of the Shoot cluster. + properties: + coreDNS: + description: CoreDNS contains the settings of the Core DNS components + running in the data plane of the Shoot cluster. + properties: + autoscaling: + description: Autoscaling contains the settings related to + autoscaling of the Core DNS components running in the data + plane of the Shoot cluster. + properties: + mode: + description: The mode of the autoscaling to be used for + the Core DNS components running in the data plane of + the Shoot cluster. Supported values are `horizontal` + and `cluster-proportional`. + type: string + required: + - mode + type: object + rewriting: + description: Rewriting contains the setting related to rewriting + of requests, which are obviously incorrect due to the unnecessary + application of the search path. + properties: + commonSuffixes: + description: CommonSuffixes are expected to be the suffix + of a fully qualified domain name. Each suffix should + contain at least one or two dots ('.') to prevent accidental + clashes. + items: + type: string + type: array + type: object + type: object + nodeLocalDNS: + description: NodeLocalDNS contains the settings of the node local + DNS components running in the data plane of the Shoot cluster. + properties: + disableForwardToUpstreamDNS: + description: DisableForwardToUpstreamDNS indicates whether + requests from node local DNS to upstream DNS should be disabled. + Default, if unspecified, is to forward requests for external + domains to upstream DNS + type: boolean + enabled: + description: Enabled indicates whether node local DNS is enabled + or not. + type: boolean + forceTCPToClusterDNS: + description: ForceTCPToClusterDNS indicates whether the connection + from the node local DNS to the cluster DNS (Core DNS) will + be forced to TCP or not. Default, if unspecified, is to + enforce TCP. + type: boolean + forceTCPToUpstreamDNS: + description: ForceTCPToUpstreamDNS indicates whether the connection + from the node local DNS to the upstream DNS (infrastructure + DNS) will be forced to TCP or not. Default, if unspecified, + is to enforce TCP. + type: boolean + required: + - enabled + type: object + type: object + tolerations: + description: Tolerations contains the tolerations for taints on seed + clusters. + items: + description: Toleration is a toleration for a seed taint. + properties: + key: + description: Key is the toleration key to be applied to a project + or shoot. + type: string + value: + description: Value is the toleration value corresponding to + the toleration key. + type: string + required: + - key + type: object + type: array + required: + - cloudProfileName + - kubernetes + - provider + - region + type: object + status: + description: Most recently observed status of the Shoot cluster. + properties: + advertisedAddresses: + description: List of addresses on which the Kube API server can be + reached. + items: + description: ShootAdvertisedAddress contains information for the + shoot's Kube API server. + properties: + name: + description: Name of the advertised address. e.g. external + type: string + url: + description: The URL of the API Server. e.g. https://api.foo.bar + or https://1.2.3.4 + type: string + required: + - name + - url + type: object + type: array + clusterIdentity: + description: ClusterIdentity is the identity of the Shoot cluster. + This field is immutable. + type: string + conditions: + description: Conditions represents the latest available observations + of a Shoots's current state. + items: + description: Condition holds the information about the state of + a resource. + properties: + codes: + description: Well-defined error codes in case the condition + reports a problem. + items: + description: ErrorCode is a string alias. + type: string + type: array + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. + format: date-time + type: string + lastUpdateTime: + description: Last time the condition was updated. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. + type: string + reason: + description: The reason for the condition's last transition. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of the condition. + type: string + required: + - lastTransitionTime + - lastUpdateTime + - message + - reason + - status + - type + type: object + type: array + constraints: + description: Constraints represents conditions of a Shoot's current + state that constraint some operations on it. + items: + description: Condition holds the information about the state of + a resource. + properties: + codes: + description: Well-defined error codes in case the condition + reports a problem. + items: + description: ErrorCode is a string alias. + type: string + type: array + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. + format: date-time + type: string + lastUpdateTime: + description: Last time the condition was updated. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. + type: string + reason: + description: The reason for the condition's last transition. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of the condition. + type: string + required: + - lastTransitionTime + - lastUpdateTime + - message + - reason + - status + - type + type: object + type: array + credentials: + description: Credentials contains information about the shoot credentials. + properties: + rotation: + description: Rotation contains information about the credential + rotations. + properties: + certificateAuthorities: + description: CertificateAuthorities contains information about + the certificate authority credential rotation. + properties: + lastCompletionTime: + description: LastCompletionTime is the most recent time + when the certificate authority credential rotation was + successfully completed. + format: date-time + type: string + lastCompletionTriggeredTime: + description: LastCompletionTriggeredTime is the recent + time when the certificate authority credential rotation + completion was triggered. + format: date-time + type: string + lastInitiationFinishedTime: + description: LastInitiationFinishedTime is the recent + time when the certificate authority credential rotation + initiation was completed. + format: date-time + type: string + lastInitiationTime: + description: LastInitiationTime is the most recent time + when the certificate authority credential rotation was + initiated. + format: date-time + type: string + phase: + description: Phase describes the phase of the certificate + authority credential rotation. + type: string + required: + - phase + type: object + etcdEncryptionKey: + description: ETCDEncryptionKey contains information about + the ETCD encryption key credential rotation. + properties: + lastCompletionTime: + description: LastCompletionTime is the most recent time + when the ETCD encryption key credential rotation was + successfully completed. + format: date-time + type: string + lastCompletionTriggeredTime: + description: LastCompletionTriggeredTime is the recent + time when the certificate authority credential rotation + completion was triggered. + format: date-time + type: string + lastInitiationFinishedTime: + description: LastInitiationFinishedTime is the recent + time when the certificate authority credential rotation + initiation was completed. + format: date-time + type: string + lastInitiationTime: + description: LastInitiationTime is the most recent time + when the ETCD encryption key credential rotation was + initiated. + format: date-time + type: string + phase: + description: Phase describes the phase of the ETCD encryption + key credential rotation. + type: string + required: + - phase + type: object + kubeconfig: + description: Kubeconfig contains information about the kubeconfig + credential rotation. + properties: + lastCompletionTime: + description: LastCompletionTime is the most recent time + when the kubeconfig credential rotation was successfully + completed. + format: date-time + type: string + lastInitiationTime: + description: LastInitiationTime is the most recent time + when the kubeconfig credential rotation was initiated. + format: date-time + type: string + type: object + observability: + description: Observability contains information about the + observability credential rotation. + properties: + lastCompletionTime: + description: LastCompletionTime is the most recent time + when the observability credential rotation was successfully + completed. + format: date-time + type: string + lastInitiationTime: + description: LastInitiationTime is the most recent time + when the observability credential rotation was initiated. + format: date-time + type: string + type: object + serviceAccountKey: + description: ServiceAccountKey contains information about + the service account key credential rotation. + properties: + lastCompletionTime: + description: LastCompletionTime is the most recent time + when the service account key credential rotation was + successfully completed. + format: date-time + type: string + lastCompletionTriggeredTime: + description: LastCompletionTriggeredTime is the recent + time when the certificate authority credential rotation + completion was triggered. + format: date-time + type: string + lastInitiationFinishedTime: + description: LastInitiationFinishedTime is the recent + time when the certificate authority credential rotation + initiation was completed. + format: date-time + type: string + lastInitiationTime: + description: LastInitiationTime is the most recent time + when the service account key credential rotation was + initiated. + format: date-time + type: string + phase: + description: Phase describes the phase of the service + account key credential rotation. + type: string + required: + - phase + type: object + sshKeypair: + description: SSHKeypair contains information about the ssh-keypair + credential rotation. + properties: + lastCompletionTime: + description: LastCompletionTime is the most recent time + when the ssh-keypair credential rotation was successfully + completed. + format: date-time + type: string + lastInitiationTime: + description: LastInitiationTime is the most recent time + when the ssh-keypair credential rotation was initiated. + format: date-time + type: string + type: object + type: object + type: object + gardener: + description: Gardener holds information about the Gardener which last + acted on the Shoot. + properties: + id: + description: ID is the Docker container id of the Gardener which + last acted on a resource. + type: string + name: + description: Name is the hostname (pod name) of the Gardener which + last acted on a resource. + type: string + version: + description: Version is the version of the Gardener which last + acted on a resource. + type: string + required: + - id + - name + - version + type: object + hibernated: + description: IsHibernated indicates whether the Shoot is currently + hibernated. + type: boolean + lastErrors: + description: LastErrors holds information about the last occurred + error(s) during an operation. + items: + description: LastError indicates the last occurred error for an + operation on a resource. + properties: + codes: + description: Well-defined error codes of the last error(s). + items: + description: ErrorCode is a string alias. + type: string + type: array + description: + description: A human readable message indicating details about + the last error. + type: string + lastUpdateTime: + description: Last time the error was reported + format: date-time + type: string + taskID: + description: ID of the task which caused this last error + type: string + required: + - description + type: object + type: array + lastHibernationTriggerTime: + description: LastHibernationTriggerTime indicates the last time when + the hibernation controller managed to change the hibernation settings + of the cluster + format: date-time + type: string + lastMaintenance: + description: LastMaintenance holds information about the last maintenance + operations on the Shoot. + properties: + description: + description: A human-readable message containing details about + the operations performed in the last maintenance. + type: string + failureReason: + description: FailureReason holds the information about the last + maintenance operation failure reason. + type: string + state: + description: Status of the last maintenance operation, one of + Processing, Succeeded, Error. + type: string + triggeredTime: + description: TriggeredTime is the time when maintenance was triggered. + format: date-time + type: string + required: + - description + - state + - triggeredTime + type: object + lastOperation: + description: LastOperation holds information about the last operation + on the Shoot. + properties: + description: + description: A human readable message indicating details about + the last operation. + type: string + lastUpdateTime: + description: Last time the operation state transitioned from one + to another. + format: date-time + type: string + progress: + description: The progress in percentage (0-100) of the last operation. + format: int32 + type: integer + state: + description: Status of the last operation, one of Aborted, Processing, + Succeeded, Error, Failed. + type: string + type: + description: Type of the last operation, one of Create, Reconcile, + Delete, Migrate, Restore. + type: string + required: + - description + - lastUpdateTime + - progress + - state + - type + type: object + migrationStartTime: + description: MigrationStartTime is the time when a migration to a + different seed was initiated. + format: date-time + type: string + observedGeneration: + description: ObservedGeneration is the most recent generation observed + for this Shoot. It corresponds to the Shoot's generation, which + is updated on mutation by the API Server. + format: int64 + type: integer + retryCycleStartTime: + description: RetryCycleStartTime is the start time of the last retry + cycle (used to determine how often an operation must be retried + until we give up). + format: date-time + type: string + seedName: + description: SeedName is the name of the seed cluster that runs the + control plane of the Shoot. This value is only written after a successful + create/reconcile operation. It will be used when control planes + are moved between Seeds. + type: string + technicalID: + description: TechnicalID is the name that is used for creating the + Seed namespace, the infrastructure resources, and basically everything + that is related to this particular Shoot. This field is immutable. + type: string + uid: + description: UID is a unique identifier for the Shoot cluster to avoid + portability between Kubernetes clusters. It is used to compute unique + hashes. This field is immutable. + type: string + required: + - gardener + - hibernated + - technicalID + - uid + type: object + type: object + served: true + storage: true diff --git a/components/kcp/config/crd/operator/operator.kyma-project.io_kymas.yaml b/components/kcp/config/crd/operator/operator.kyma-project.io_kymas.yaml new file mode 100644 index 000000000..b511ebfe1 --- /dev/null +++ b/components/kcp/config/crd/operator/operator.kyma-project.io_kymas.yaml @@ -0,0 +1,806 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.13.0 + name: kymas.operator.kyma-project.io +spec: + group: operator.kyma-project.io + names: + kind: Kyma + listKind: KymaList + plural: kymas + singular: kyma + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.state + name: State + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + deprecated: true + deprecationWarning: kyma-project.io/v1beta1 Kyma is deprecated. Use v1beta2 instead. + name: v1beta1 + schema: + openAPIV3Schema: + description: Kyma is the Schema for the kymas API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: KymaSpec defines the desired state of Kyma. + properties: + channel: + description: Channel specifies the desired Channel of the Installation, + usually targeting different module versions. + maxLength: 32 + minLength: 3 + pattern: ^[a-z]+$ + type: string + modules: + description: Modules specifies the list of modules to be installed + items: + description: Module defines the components to be installed. + properties: + channel: + description: Channel is the desired channel of the Module. If + this changes or is set, it will be used to resolve a new ModuleTemplate + based on the new resolved resources. + maxLength: 32 + minLength: 3 + pattern: ^[a-z]+$ + type: string + controller: + description: ControllerName is able to set the controller used + for reconciliation of the module. It can be used together + with Cache Configuration on the Operator responsible for the + templated Modules to split workload. + type: string + customResourcePolicy: + default: CreateAndDelete + description: CustomResourcePolicy determines how a ModuleTemplate + should be parsed. When CustomResourcePolicy is set to CustomResourcePolicyCreateAndDelete, + the Manifest will receive instructions to create it on installation + with the default values provided in ModuleTemplate, and to + remove it when the module or Kyma is deleted. + enum: + - CreateAndDelete + - Ignore + type: string + name: + description: "Name is a unique identifier of the module. It + is used to resolve a ModuleTemplate for creating a set of + resources on the cluster. \n Name can be one of 3 kinds: - + The ModuleName label value of the module-template, e.g. operator.kyma-project.io/module-name=my-module + - The Name or Namespace/Name of a ModuleTemplate, e.g. my-moduletemplate + or kyma-system/my-moduletemplate - The FQDN, e.g. kyma-project.io/module/my-module + as located in .spec.descriptor.component.name" + type: string + remoteModuleTemplateRef: + description: RemoteModuleTemplateRef is the reference (FQDN, + Namespace/Name, Module Name Label) to the module template + on the remote cluster. If specified, the module template will + be fetched from the SKR and reconciled. + type: string + required: + - name + type: object + type: array + sync: + description: Active Synchronization Settings + properties: + enabled: + default: false + description: Enabled set to true will look up a kubeconfig for + the remote cluster based on the strategy and synchronize its + state there. + type: boolean + moduleCatalog: + default: true + description: ModuleCatalog set to true will cause a copy of all + ModuleTemplate in the cluster to be synchronized for discovery + purposes + type: boolean + namespace: + description: The target namespace, if empty the namespace is reflected + from the control plane Note that cleanup is currently not supported + if you are switching the namespace, so you will manually need + to clean up old synchronized Kymas + type: string + noModuleCopy: + default: true + description: NoModuleCopy set to true will cause the remote Kyma + to be initialized without copying over the module spec of the + control plane into the SKR + type: boolean + strategy: + default: secret + description: Strategy determines the way to look up the remotely + synced kubeconfig, by default it is fetched from a secret + type: string + type: object + required: + - channel + type: object + status: + description: KymaStatus defines the observed state of Kyma + properties: + activeChannel: + description: Active Channel + type: string + conditions: + description: List of status conditions to indicate the status of a + ServiceInstance. + items: + description: "Condition contains details for one aspect of the current + state of this API Resource. --- This struct is intended for direct + use as an array at the field path .status.conditions. For example, + \n type FooStatus struct{ // Represents the observations of a + foo's current state. // Known .status.conditions.type are: \"Available\", + \"Progressing\", and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge + // +listType=map // +listMapKey=type Conditions []metav1.Condition + `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" + protobuf:\"bytes,1,rep,name=conditions\"` \n // other fields }" + properties: + lastTransitionTime: + description: lastTransitionTime is the last time the condition + transitioned from one status to another. This should be when + the underlying condition changed. If that is not known, then + using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: message is a human readable message indicating + details about the transition. This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: observedGeneration represents the .metadata.generation + that the condition was set based upon. For instance, if .metadata.generation + is currently 12, but the .status.conditions[x].observedGeneration + is 9, the condition is out of date with respect to the current + state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: reason contains a programmatic identifier indicating + the reason for the condition's last transition. Producers + of specific condition types may define expected values and + meanings for this field, and whether the values are considered + a guaranteed API. The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + --- Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + lastOperation: + description: LastOperation defines the last operation from the control-loop. + properties: + lastUpdateTime: + format: date-time + type: string + operation: + type: string + required: + - operation + type: object + modules: + description: Contains essential information about the current deployed + module + items: + properties: + channel: + description: Channel tracks the active Channel of the Module. + In Case it changes, the new Channel will have caused a new + lookup to be necessary that maybe picks a different ModuleTemplate, + which is why we need to reconcile. + type: string + fqdn: + description: FQDN is the fully qualified domain name of the + module. In the ModuleTemplate it is located in .spec.descriptor.component.name + of the ModuleTemplate FQDN is used to calculate Namespace + and Name of the Manifest for tracking. + type: string + manifest: + description: Manifest contains the Information of a related + Manifest + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of + this representation of an object. Servers should convert + recognized schemas to the latest internal value, and may + reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST + resource this object represents. Servers may infer this + from the endpoint the client submits requests to. Cannot + be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + description: PartialMeta is a subset of ObjectMeta that + contains relevant information to track an Object. see + https://github.com/kubernetes/apimachinery/blob/v0.26.1/pkg/apis/meta/v1/types.go#L111 + properties: + generation: + description: A sequence number representing a specific + generation of the desired state. Populated by the + system. Read-only. + format: int64 + type: integer + name: + description: 'Name must be unique within a namespace. + Is required when creating resources, although some + resources may allow a client to request the generation + of an appropriate name automatically. Name is primarily + intended for creation idempotence and configuration + definition. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/identifiers#names' + type: string + namespace: + description: "Namespace defines the space within which + each name must be unique. An empty namespace is equivalent + to the \"default\" namespace, but \"default\" is the + canonical representation. Not all objects are required + to be scoped to a namespace - the value of this field + for those objects will be empty. \n Must be a DNS_LABEL. + Cannot be updated. More info: http://kubernetes.io/docs/user-guide/namespaces" + type: string + type: object + type: object + message: + description: Message is a human-readable message indicating + details about the State. + type: string + name: + description: Name defines the name of the Module in the Spec + that the status is used for. It can be any kind of Reference + format supported by Module.Name. + type: string + resource: + description: Resource contains information about the created + module CR. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of + this representation of an object. Servers should convert + recognized schemas to the latest internal value, and may + reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST + resource this object represents. Servers may infer this + from the endpoint the client submits requests to. Cannot + be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + description: PartialMeta is a subset of ObjectMeta that + contains relevant information to track an Object. see + https://github.com/kubernetes/apimachinery/blob/v0.26.1/pkg/apis/meta/v1/types.go#L111 + properties: + generation: + description: A sequence number representing a specific + generation of the desired state. Populated by the + system. Read-only. + format: int64 + type: integer + name: + description: 'Name must be unique within a namespace. + Is required when creating resources, although some + resources may allow a client to request the generation + of an appropriate name automatically. Name is primarily + intended for creation idempotence and configuration + definition. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/identifiers#names' + type: string + namespace: + description: "Namespace defines the space within which + each name must be unique. An empty namespace is equivalent + to the \"default\" namespace, but \"default\" is the + canonical representation. Not all objects are required + to be scoped to a namespace - the value of this field + for those objects will be empty. \n Must be a DNS_LABEL. + Cannot be updated. More info: http://kubernetes.io/docs/user-guide/namespaces" + type: string + type: object + type: object + state: + description: State of the Module in the currently tracked Generation + enum: + - Processing + - Deleting + - Ready + - Error + - "" + - Warning + type: string + template: + description: It contains information about the last parsed ModuleTemplate + in Context of the Installation. This will update when Channel + or the ModuleTemplate is changed. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of + this representation of an object. Servers should convert + recognized schemas to the latest internal value, and may + reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST + resource this object represents. Servers may infer this + from the endpoint the client submits requests to. Cannot + be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + description: PartialMeta is a subset of ObjectMeta that + contains relevant information to track an Object. see + https://github.com/kubernetes/apimachinery/blob/v0.26.1/pkg/apis/meta/v1/types.go#L111 + properties: + generation: + description: A sequence number representing a specific + generation of the desired state. Populated by the + system. Read-only. + format: int64 + type: integer + name: + description: 'Name must be unique within a namespace. + Is required when creating resources, although some + resources may allow a client to request the generation + of an appropriate name automatically. Name is primarily + intended for creation idempotence and configuration + definition. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/identifiers#names' + type: string + namespace: + description: "Namespace defines the space within which + each name must be unique. An empty namespace is equivalent + to the \"default\" namespace, but \"default\" is the + canonical representation. Not all objects are required + to be scoped to a namespace - the value of this field + for those objects will be empty. \n Must be a DNS_LABEL. + Cannot be updated. More info: http://kubernetes.io/docs/user-guide/namespaces" + type: string + type: object + type: object + version: + description: Channel tracks the active Version of the Module. + type: string + required: + - name + - state + type: object + type: array + state: + description: State signifies current state of Kyma. Value can be one + of ("Ready", "Processing", "Error", "Deleting"). + enum: + - Processing + - Deleting + - Ready + - Error + - "" + - Warning + type: string + type: object + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - jsonPath: .status.state + name: State + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: Kyma is the Schema for the kymas API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: KymaSpec defines the desired state of Kyma. + properties: + channel: + description: Channel specifies the desired Channel of the Installation, + usually targeting different module versions. + maxLength: 32 + minLength: 3 + pattern: ^[a-z]+$ + type: string + modules: + description: Modules specifies the list of modules to be installed + items: + description: Module defines the components to be installed. + properties: + channel: + description: Channel is the desired channel of the Module. If + this changes or is set, it will be used to resolve a new ModuleTemplate + based on the new resolved resources. + maxLength: 32 + minLength: 3 + pattern: ^[a-z]+$ + type: string + controller: + description: ControllerName is able to set the controller used + for reconciliation of the module. It can be used together + with Cache Configuration on the Operator responsible for the + templated Modules to split workload. + type: string + customResourcePolicy: + default: CreateAndDelete + description: CustomResourcePolicy determines how a ModuleTemplate + should be parsed. When CustomResourcePolicy is set to CustomResourcePolicyCreateAndDelete, + the Manifest will receive instructions to create it on installation + with the default values provided in ModuleTemplate, and to + remove it when the module or Kyma is deleted. + enum: + - CreateAndDelete + - Ignore + type: string + name: + description: "Name is a unique identifier of the module. It + is used to resolve a ModuleTemplate for creating a set of + resources on the cluster. \n Name can be one of 3 kinds: - + The ModuleName label value of the module-template, e.g. operator.kyma-project.io/module-name=my-module + - The Name or Namespace/Name of a ModuleTemplate, e.g. my-moduletemplate + or kyma-system/my-moduletemplate - The FQDN, e.g. kyma-project.io/module/my-module + as located in .spec.descriptor.component.name" + type: string + remoteModuleTemplateRef: + description: RemoteModuleTemplateRef is the reference (FQDN, + Namespace/Name, Module Name Label) to the module template + on the remote cluster. If specified, the module template will + be fetched from the SKR and reconciled. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - channel + type: object + status: + description: KymaStatus defines the observed state of Kyma + properties: + activeChannel: + description: Active Channel + type: string + conditions: + description: List of status conditions to indicate the status of a + ServiceInstance. + items: + description: "Condition contains details for one aspect of the current + state of this API Resource. --- This struct is intended for direct + use as an array at the field path .status.conditions. For example, + \n type FooStatus struct{ // Represents the observations of a + foo's current state. // Known .status.conditions.type are: \"Available\", + \"Progressing\", and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge + // +listType=map // +listMapKey=type Conditions []metav1.Condition + `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" + protobuf:\"bytes,1,rep,name=conditions\"` \n // other fields }" + properties: + lastTransitionTime: + description: lastTransitionTime is the last time the condition + transitioned from one status to another. This should be when + the underlying condition changed. If that is not known, then + using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: message is a human readable message indicating + details about the transition. This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: observedGeneration represents the .metadata.generation + that the condition was set based upon. For instance, if .metadata.generation + is currently 12, but the .status.conditions[x].observedGeneration + is 9, the condition is out of date with respect to the current + state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: reason contains a programmatic identifier indicating + the reason for the condition's last transition. Producers + of specific condition types may define expected values and + meanings for this field, and whether the values are considered + a guaranteed API. The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + --- Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + lastOperation: + description: LastOperation defines the last operation from the control-loop. + properties: + lastUpdateTime: + format: date-time + type: string + operation: + type: string + required: + - operation + type: object + modules: + description: Contains essential information about the current deployed + module + items: + properties: + channel: + description: Channel tracks the active Channel of the Module. + In Case it changes, the new Channel will have caused a new + lookup to be necessary that maybe picks a different ModuleTemplate, + which is why we need to reconcile. + type: string + fqdn: + description: FQDN is the fully qualified domain name of the + module. In the ModuleTemplate it is located in .spec.descriptor.component.name + of the ModuleTemplate FQDN is used to calculate Namespace + and Name of the Manifest for tracking. + type: string + manifest: + description: Manifest contains the Information of a related + Manifest + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of + this representation of an object. Servers should convert + recognized schemas to the latest internal value, and may + reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST + resource this object represents. Servers may infer this + from the endpoint the client submits requests to. Cannot + be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + description: PartialMeta is a subset of ObjectMeta that + contains relevant information to track an Object. see + https://github.com/kubernetes/apimachinery/blob/v0.26.1/pkg/apis/meta/v1/types.go#L111 + properties: + generation: + description: A sequence number representing a specific + generation of the desired state. Populated by the + system. Read-only. + format: int64 + type: integer + name: + description: 'Name must be unique within a namespace. + Is required when creating resources, although some + resources may allow a client to request the generation + of an appropriate name automatically. Name is primarily + intended for creation idempotence and configuration + definition. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/identifiers#names' + type: string + namespace: + description: "Namespace defines the space within which + each name must be unique. An empty namespace is equivalent + to the \"default\" namespace, but \"default\" is the + canonical representation. Not all objects are required + to be scoped to a namespace - the value of this field + for those objects will be empty. \n Must be a DNS_LABEL. + Cannot be updated. More info: http://kubernetes.io/docs/user-guide/namespaces" + type: string + type: object + type: object + message: + description: Message is a human-readable message indicating + details about the State. + type: string + name: + description: Name defines the name of the Module in the Spec + that the status is used for. It can be any kind of Reference + format supported by Module.Name. + type: string + resource: + description: Resource contains information about the created + module CR. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of + this representation of an object. Servers should convert + recognized schemas to the latest internal value, and may + reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST + resource this object represents. Servers may infer this + from the endpoint the client submits requests to. Cannot + be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + description: PartialMeta is a subset of ObjectMeta that + contains relevant information to track an Object. see + https://github.com/kubernetes/apimachinery/blob/v0.26.1/pkg/apis/meta/v1/types.go#L111 + properties: + generation: + description: A sequence number representing a specific + generation of the desired state. Populated by the + system. Read-only. + format: int64 + type: integer + name: + description: 'Name must be unique within a namespace. + Is required when creating resources, although some + resources may allow a client to request the generation + of an appropriate name automatically. Name is primarily + intended for creation idempotence and configuration + definition. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/identifiers#names' + type: string + namespace: + description: "Namespace defines the space within which + each name must be unique. An empty namespace is equivalent + to the \"default\" namespace, but \"default\" is the + canonical representation. Not all objects are required + to be scoped to a namespace - the value of this field + for those objects will be empty. \n Must be a DNS_LABEL. + Cannot be updated. More info: http://kubernetes.io/docs/user-guide/namespaces" + type: string + type: object + type: object + state: + description: State of the Module in the currently tracked Generation + enum: + - Processing + - Deleting + - Ready + - Error + - "" + - Warning + type: string + template: + description: It contains information about the last parsed ModuleTemplate + in Context of the Installation. This will update when Channel + or the ModuleTemplate is changed. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of + this representation of an object. Servers should convert + recognized schemas to the latest internal value, and may + reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST + resource this object represents. Servers may infer this + from the endpoint the client submits requests to. Cannot + be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + description: PartialMeta is a subset of ObjectMeta that + contains relevant information to track an Object. see + https://github.com/kubernetes/apimachinery/blob/v0.26.1/pkg/apis/meta/v1/types.go#L111 + properties: + generation: + description: A sequence number representing a specific + generation of the desired state. Populated by the + system. Read-only. + format: int64 + type: integer + name: + description: 'Name must be unique within a namespace. + Is required when creating resources, although some + resources may allow a client to request the generation + of an appropriate name automatically. Name is primarily + intended for creation idempotence and configuration + definition. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/identifiers#names' + type: string + namespace: + description: "Namespace defines the space within which + each name must be unique. An empty namespace is equivalent + to the \"default\" namespace, but \"default\" is the + canonical representation. Not all objects are required + to be scoped to a namespace - the value of this field + for those objects will be empty. \n Must be a DNS_LABEL. + Cannot be updated. More info: http://kubernetes.io/docs/user-guide/namespaces" + type: string + type: object + type: object + version: + description: Channel tracks the active Version of the Module. + type: string + required: + - name + - state + type: object + type: array + state: + description: State signifies current state of Kyma. Value can be one + of ("Ready", "Processing", "Error", "Deleting"). + enum: + - Processing + - Deleting + - Ready + - Error + - "" + - Warning + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/components/kcp/internal/controller/cloud-control/iprange_controller.go b/components/kcp/internal/controller/cloud-control/iprange_controller.go index 65c1c2f96..5740bc022 100644 --- a/components/kcp/internal/controller/cloud-control/iprange_controller.go +++ b/components/kcp/internal/controller/cloud-control/iprange_controller.go @@ -34,25 +34,33 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" ) -func NewIpRangeReconciler( - mgr manager.Manager, +func SetupIpRangeReconciler( + kcpManager manager.Manager, awsProvider awsclient.SkrClientProvider[iprangeclient.Client], gcpSvcNetProvider gcpclient.ClientProvider[gcpiprangeclient.ServiceNetworkingClient], gcpComputeProvider gcpclient.ClientProvider[gcpiprangeclient.ComputeClient], -) *IpRangeReconciler { - return &IpRangeReconciler{ - Reconciler: iprange.NewIPRangeReconciler( - composed.NewStateFactory(composed.NewStateClusterFromManager(mgr)), +) error { + return NewIpRangeReconciler( + iprange.NewIPRangeReconciler( + composed.NewStateFactory(composed.NewStateClusterFromManager(kcpManager)), focal.NewStateFactory(), awsiprange.NewStateFactory(awsProvider, abstractions.NewOSEnvironment()), azureiprange.NewStateFactory(nil), gcpiprange.NewStateFactory(gcpSvcNetProvider, gcpComputeProvider, abstractions.NewOSEnvironment()), ), + ).SetupWithManager(kcpManager) +} + +func NewIpRangeReconciler( + reconciler iprange.IPRangeReconciler, +) *IpRangeReconciler { + return &IpRangeReconciler{ + Reconciler: reconciler, } } type IpRangeReconciler struct { - Reconciler *iprange.IPRangeReconciler + Reconciler iprange.IPRangeReconciler } //+kubebuilder:rbac:groups=cloud-control.kyma-project.io,resources=ipranges,verbs=get;list;watch;create;update;patch;delete @@ -61,15 +69,11 @@ type IpRangeReconciler struct { // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. -// TODO(user): Modify the Reconcile function to compare the state specified by -// the IpRange object against the actual cluster state, and then -// perform operations to make the cluster state reflect the state specified by -// the user. // // For more details, check Reconcile and its Result here: // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.16.3/pkg/reconcile func (r *IpRangeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - return r.Reconciler.Run(ctx, req) + return r.Reconciler.Reconcile(ctx, req) } // SetupWithManager sets up the controller with the Manager. diff --git a/components/kcp/internal/controller/cloud-control/nfsinstance_controller.go b/components/kcp/internal/controller/cloud-control/nfsinstance_controller.go index 4226d2cea..296511232 100644 --- a/components/kcp/internal/controller/cloud-control/nfsinstance_controller.go +++ b/components/kcp/internal/controller/cloud-control/nfsinstance_controller.go @@ -34,24 +34,32 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" ) -func NewNfsInstanceReconciler( - mgr manager.Manager, +func SetupNfsInstanceReconciler( + kcpManager manager.Manager, awsSkrProvider awsclient.SkrClientProvider[awsnfsinstanceclient.Client], filestoreClientProvider gcpclient.ClientProvider[gcpnfsinstanceclient.FilestoreClient], -) *NfsInstanceReconciler { - return &NfsInstanceReconciler{ - Reconciler: nfsinstance.NewNfsInstanceReconciler( - composed.NewStateFactory(composed.NewStateClusterFromManager(mgr)), +) error { + return NewNfsInstanceReconciler( + nfsinstance.NewNfsInstanceReconciler( + composed.NewStateFactory(composed.NewStateClusterFromManager(kcpManager)), focal.NewStateFactory(), awsnfsinstance.NewStateFactory(awsSkrProvider, abstractions.NewOSEnvironment()), azurenfsinstance.NewStateFactory(), gcpnfsinstance.NewStateFactory(filestoreClientProvider, abstractions.NewOSEnvironment()), ), + ).SetupWithManager(kcpManager) +} + +func NewNfsInstanceReconciler( + reconciler nfsinstance.NfsInstanceReconciler, +) *NfsInstanceReconciler { + return &NfsInstanceReconciler{ + Reconciler: reconciler, } } type NfsInstanceReconciler struct { - Reconciler *nfsinstance.NfsInstanceReconciler + Reconciler nfsinstance.NfsInstanceReconciler } //+kubebuilder:rbac:groups=cloud-control.kyma-project.io,resources=nfsinstances,verbs=get;list;watch;create;update;patch;delete @@ -68,7 +76,7 @@ type NfsInstanceReconciler struct { // For more details, check Reconcile and its Result here: // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.16.3/pkg/reconcile func (r *NfsInstanceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - return r.Reconciler.Run(ctx, req) + return r.Reconciler.Reconcile(ctx, req) } // SetupWithManager sets up the controller with the Manager. diff --git a/components/kcp/internal/controller/cloud-control/scope_aws_test.go b/components/kcp/internal/controller/cloud-control/scope_aws_test.go new file mode 100644 index 000000000..09d8d663c --- /dev/null +++ b/components/kcp/internal/controller/cloud-control/scope_aws_test.go @@ -0,0 +1,59 @@ +package cloudresources + +// +//import ( +// cloudcontrolv1beta1 "github.com/kyma-project/cloud-manager/components/kcp/api/cloud-control/v1beta1" +// "github.com/kyma-project/cloud-manager/components/kcp/pkg/util" +// . "github.com/onsi/ginkgo/v2" +// . "github.com/onsi/gomega" +// "k8s.io/apimachinery/pkg/types" +// "sigs.k8s.io/controller-runtime/pkg/client" +// "time" +//) +// +//var _ = Describe("Scope AWS", func() { +// +// const ( +// kymaName = "5d60be8c-e422-48ff-bd0a-166b0e09dc58" +// +// timeout = time.Second * 5 +// duration = time.Second * 5 +// interval = time.Millisecond * 250 +// ) +// +// It("Scope lifecycle", func() { +// kcpObjKey := types.NamespacedName{ +// Namespace: infra.KCP().Namespace(), +// Name: kymaName, +// } +// +// Expect(infra.GivenGardenShootAwsExists(kymaName)). +// NotTo(HaveOccurred(), "failed creating garden shoot for aws") +// +// Expect(infra.GivenKymaCRExists(kymaName)). +// NotTo(HaveOccurred(), "failed creating kyma cr") +// +// scope := &cloudcontrolv1beta1.Scope{} +// +// Consistently(func() (exists bool, err error) { +// err = infra.KCP().Client().Get(infra.Ctx(), kcpObjKey, scope) +// exists = err == nil +// return exists, client.IgnoreNotFound(err) +// }, duration, interval). +// Should(BeFalse(), "expected Scope not to exist") +// +// Expect(infra.WhenKymaModuleStateUpdates(kymaName, util.KymaModuleStateReady)). +// NotTo(HaveOccurred()) +// +// Eventually(func() (exists bool, err error) { +// err = infra.KCP().Client().Get(infra.Ctx(), kcpObjKey, scope) +// exists = err == nil +// return exists, client.IgnoreNotFound(err) +// }, timeout, interval). +// Should(BeTrue(), "expected Scope to be created") +// +// Expect(scope).NotTo(BeNil()) +// Expect(scope.Spec.Provider).To(Equal(string(cloudcontrolv1beta1.ProviderAws))) +// }) +// +//}) diff --git a/components/kcp/internal/controller/cloud-control/scope_controller.go b/components/kcp/internal/controller/cloud-control/scope_controller.go index e24d61cd6..c2f6693a1 100644 --- a/components/kcp/internal/controller/cloud-control/scope_controller.go +++ b/components/kcp/internal/controller/cloud-control/scope_controller.go @@ -3,12 +3,11 @@ package cloudresources import ( "context" cloudcontrolv1beta1 "github.com/kyma-project/cloud-manager/components/kcp/api/cloud-control/v1beta1" - "github.com/kyma-project/cloud-manager/components/kcp/pkg/common/abstractions" kcpscope "github.com/kyma-project/cloud-manager/components/kcp/pkg/kcp/scope" scopeclient "github.com/kyma-project/cloud-manager/components/kcp/pkg/kcp/scope/client" awsclient "github.com/kyma-project/cloud-manager/components/kcp/pkg/provider/aws/client" + skrruntime "github.com/kyma-project/cloud-manager/components/kcp/pkg/skr/runtime" "github.com/kyma-project/cloud-manager/components/kcp/pkg/util" - "github.com/kyma-project/cloud-manager/components/lib/composed" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" @@ -19,13 +18,25 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" ) -func NewScopeReconciler(mgr manager.Manager, awsStsClientProvider awsclient.GardenClientProvider[scopeclient.AwsStsClient]) *ScopeReconciler { - return &ScopeReconciler{ - Reconciler: kcpscope.NewScopeReconciler(kcpscope.NewStateFactory( - composed.NewStateFactory(composed.NewStateClusterFromManager(mgr)), - abstractions.NewFileReader(), +func SetupScopeReconciler( + kcpManager manager.Manager, + awsStsClientProvider awsclient.GardenClientProvider[scopeclient.AwsStsClient], + activeSkrCollection skrruntime.ActiveSkrCollection, +) error { + return NewScopeReconciler( + kcpscope.New( + kcpManager, awsStsClientProvider, - )), + activeSkrCollection, + ), + ).SetupWithManager(kcpManager) +} + +func NewScopeReconciler( + reconciler kcpscope.ScopeReconciler, +) *ScopeReconciler { + return &ScopeReconciler{ + Reconciler: reconciler, } } diff --git a/components/kcp/internal/controller/cloud-control/suite_test.go b/components/kcp/internal/controller/cloud-control/suite_test.go index 6f1674f01..a55630ed4 100644 --- a/components/kcp/internal/controller/cloud-control/suite_test.go +++ b/components/kcp/internal/controller/cloud-control/suite_test.go @@ -17,31 +17,26 @@ limitations under the License. package cloudresources import ( - "fmt" - "path/filepath" - "runtime" + "context" + "github.com/kyma-project/cloud-manager/components/kcp/pkg/testinfra" "testing" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "k8s.io/client-go/kubernetes/scheme" - "k8s.io/client-go/rest" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/envtest" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" - - cloudcontrolv1beta1 "github.com/kyma-project/cloud-manager/components/kcp/api/cloud-control/v1beta1" //+kubebuilder:scaffold:imports ) // These tests use Ginkgo (BDD-style Go testing framework). Refer to // http://onsi.github.io/ginkgo/ to learn more about Ginkgo. -var cfg *rest.Config -var k8sClient client.Client -var testEnv *envtest.Environment +//var cfg *rest.Config +//var k8sClient client.Client +//var testEnv *envtest.Environment + +var infra testinfra.Infra func TestControllers(t *testing.T) { RegisterFailHandler(Fail) @@ -53,38 +48,31 @@ var _ = BeforeSuite(func() { logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) By("bootstrapping test environment") - testEnv = &envtest.Environment{ - CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "config", "crd", "bases")}, - ErrorIfCRDPathMissing: true, - - // The BinaryAssetsDirectory is only required if you want to run the tests directly - // without call the makefile target test. If not informed it will look for the - // default path defined in controller-runtime which is /usr/local/kubebuilder/. - // Note that you must have the required binaries setup under the bin directory to perform - // the tests directly. When we run make test it will be setup and used automatically. - BinaryAssetsDirectory: filepath.Join("..", "..", "..", "bin", "k8s", - fmt.Sprintf("1.28.3-%s-%s", runtime.GOOS, runtime.GOARCH)), - } - var err error - // cfg is defined in this file globally. - cfg, err = testEnv.Start() - Expect(err).NotTo(HaveOccurred()) - Expect(cfg).NotTo(BeNil()) - - err = cloudcontrolv1beta1.AddToScheme(scheme.Scheme) - Expect(err).NotTo(HaveOccurred()) - - //+kubebuilder:scaffold:scheme - - k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) - Expect(err).NotTo(HaveOccurred()) - Expect(k8sClient).NotTo(BeNil()) - + infra, err = testinfra.Start() + Expect(err). + NotTo(HaveOccurred(), "failed starting infra clusters") + + Expect(infra.KCP().GivenNamespaceExists(infra.KCP().Namespace())). + NotTo(HaveOccurred(), "failed creating namespace %s in KCP", infra.KCP().Namespace()) + Expect(infra.SKR().GivenNamespaceExists(infra.SKR().Namespace())). + NotTo(HaveOccurred(), "failed creating namespace %s in SKR", infra.SKR().Namespace()) + Expect(infra.Garden().GivenNamespaceExists(infra.Garden().Namespace())). + NotTo(HaveOccurred(), "failed creating namespace %s in Garden", infra.Garden().Namespace()) + + // Setup controllers + Expect(SetupScopeReconciler( + infra.KcpManager(), + infra.AwsMock().ScopeGardenProvider(), + infra.Looper(), + )).NotTo(HaveOccurred()) + + // Start controllers + infra.StartControllers(context.Background()) }) var _ = AfterSuite(func() { By("tearing down the test environment") - err := testEnv.Stop() + err := infra.Stop() Expect(err).NotTo(HaveOccurred()) }) diff --git a/components/kcp/internal/controller/cloud-control/vpcpeering_controller.go b/components/kcp/internal/controller/cloud-control/vpcpeering_controller.go index 0c67ae46c..91af2e76b 100644 --- a/components/kcp/internal/controller/cloud-control/vpcpeering_controller.go +++ b/components/kcp/internal/controller/cloud-control/vpcpeering_controller.go @@ -29,6 +29,14 @@ import ( cloudresourcesv1beta1 "github.com/kyma-project/cloud-manager/components/kcp/api/cloud-control/v1beta1" ) +func SetupVpcPeeringReconciler( + kcpManager manager.Manager, +) error { + return NewVpcPeeringReconciler( + kcpManager, + ).SetupWithManager(kcpManager) +} + func NewVpcPeeringReconciler(mgr manager.Manager) *VpcPeeringReconciler { return &VpcPeeringReconciler{ Client: mgr.GetClient(), diff --git a/components/kcp/pkg/iprange/reconciler.go b/components/kcp/pkg/iprange/reconciler.go index 6f1d3d717..b210be529 100644 --- a/components/kcp/pkg/iprange/reconciler.go +++ b/components/kcp/pkg/iprange/reconciler.go @@ -2,6 +2,7 @@ package iprange import ( "context" + "sigs.k8s.io/controller-runtime/pkg/reconcile" cloudresourcesv1beta1 "github.com/kyma-project/cloud-manager/components/kcp/api/cloud-control/v1beta1" "github.com/kyma-project/cloud-manager/components/kcp/pkg/common/actions/focal" @@ -13,7 +14,11 @@ import ( ctrl "sigs.k8s.io/controller-runtime" ) -type IPRangeReconciler struct { +type IPRangeReconciler interface { + reconcile.Reconciler +} + +type ipRangeReconciler struct { composedStateFactory composed.StateFactory focalStateFactory focal.StateFactory @@ -28,8 +33,8 @@ func NewIPRangeReconciler( awsStateFactory awsiprange.StateFactory, azureStateFactory azureiprange.StateFactory, gcpStateFactory gcpiprange.StateFactory, -) *IPRangeReconciler { - return &IPRangeReconciler{ +) IPRangeReconciler { + return &ipRangeReconciler{ composedStateFactory: composedStateFactory, focalStateFactory: focalStateFactory, awsStateFactory: awsStateFactory, @@ -38,14 +43,14 @@ func NewIPRangeReconciler( } } -func (r *IPRangeReconciler) Run(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { +func (r *ipRangeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { state := r.newFocalState(req.NamespacedName) action := r.newAction() return composed.Handle(action(ctx, state)) } -func (r *IPRangeReconciler) newAction() composed.Action { +func (r *ipRangeReconciler) newAction() composed.Action { return composed.ComposeActions( "main", focal.New(), @@ -67,7 +72,7 @@ func (r *IPRangeReconciler) newAction() composed.Action { ) } -func (r *IPRangeReconciler) newFocalState(name types.NamespacedName) focal.State { +func (r *ipRangeReconciler) newFocalState(name types.NamespacedName) focal.State { return r.focalStateFactory.NewState( r.composedStateFactory.NewState(name, &cloudresourcesv1beta1.IpRange{}), ) diff --git a/components/kcp/pkg/kcp/scope/addKymaFinalizer.go b/components/kcp/pkg/kcp/scope/addKymaFinalizer.go index 2c4ab13e2..d14b82b11 100644 --- a/components/kcp/pkg/kcp/scope/addKymaFinalizer.go +++ b/components/kcp/pkg/kcp/scope/addKymaFinalizer.go @@ -28,7 +28,7 @@ func addKymaFinalizer(ctx context.Context, st composed.State) (error, context.Co err := state.Cluster().K8sClient().Update(ctx, state.kyma) if err != nil { - return composed.LogErrorAndReturn(err, "Error updating Kyma CR with added finalizer", composed.StopWithRequeueDelay(time.Second), nil) + return composed.LogErrorAndReturn(err, "Error updating Kyma CR with added finalizer", composed.StopWithRequeueDelay(time.Second), ctx) } return composed.StopWithRequeueDelay(time.Second), nil diff --git a/components/kcp/pkg/kcp/scope/createGardenerClient.go b/components/kcp/pkg/kcp/scope/createGardenerClient.go index 7bfec0d6d..3b08ebaa9 100644 --- a/components/kcp/pkg/kcp/scope/createGardenerClient.go +++ b/components/kcp/pkg/kcp/scope/createGardenerClient.go @@ -4,6 +4,8 @@ import ( "context" gardenerClient "github.com/gardener/gardener/pkg/client/core/clientset/versioned/typed/core/v1beta1" "github.com/kyma-project/cloud-manager/components/lib/composed" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" kubernetesClient "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/clientcmd" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" @@ -13,26 +15,31 @@ import ( func createGardenerClient(ctx context.Context, st composed.State) (error, context.Context) { logger := composed.LoggerFromCtx(ctx) state := st.(*State) - fn := os.Getenv("GARDENER_CREDENTIALS") - if len(fn) == 0 { - fn = "/opt/cloud-manager/gardener-credentials/kubeconfig" - } - logger = logger.WithValues("credentialsPath", fn) logger.Info("Loading gardener credentials") - kubeBytes, err := state.fileReader.ReadFile(fn) + + secret := &corev1.Secret{} + err := state.Cluster().K8sClient().Get(ctx, types.NamespacedName{ + Namespace: "kcp-system", + Name: "gardener-credentials", + }, secret) if err != nil { - return composed.LogErrorAndReturn(err, "Error loading gardener client", composed.StopAndForget, nil) + return composed.LogErrorAndReturn(err, "Error getting gardener credentials", composed.StopWithRequeue, ctx) + } + + kubeBytes, ok := secret.Data["kubeconfig"] + if !ok { + return composed.LogErrorAndReturn(err, "Gardener credentials missing kubeconfig", composed.StopAndForget, ctx) } config, err := clientcmd.NewClientConfigFromBytes(kubeBytes) if err != nil { - return composed.LogErrorAndReturn(err, "Error creating gardener client config", composed.StopAndForget, nil) + return composed.LogErrorAndReturn(err, "Error creating gardener client config", composed.StopAndForget, ctx) } rawConfig, err := config.RawConfig() if err != nil { - return composed.LogErrorAndReturn(err, "Error getting gardener raw client config", composed.StopAndForget, nil) + return composed.LogErrorAndReturn(err, "Error getting gardener raw client config", composed.StopAndForget, ctx) } var configContext *clientcmdapi.Context @@ -50,24 +57,24 @@ func createGardenerClient(ctx context.Context, st composed.State) (error, contex state.shootNamespace = os.Getenv("GARDENER_NAMESPACE") } - logger = logger.WithValues("shootProject", state.shootNamespace) + logger = logger.WithValues("shootNamespace", state.shootNamespace) logger.Info("Detected shoot namespace") restConfig, err := clientcmd.RESTConfigFromKubeConfig(kubeBytes) if err != nil { - return composed.LogErrorAndReturn(err, "Error creating gardener rest config", composed.StopAndForget, nil) + return composed.LogErrorAndReturn(err, "Error creating gardener rest config", composed.StopAndForget, ctx) } gClient, err := gardenerClient.NewForConfig(restConfig) if err != nil { - return composed.LogErrorAndReturn(err, "Error creating gardener client", composed.StopAndForget, nil) + return composed.LogErrorAndReturn(err, "Error creating gardener client", composed.StopAndForget, ctx) } state.gardenerClient = gClient k8sClient, err := kubernetesClient.NewForConfig(restConfig) if err != nil { - return composed.LogErrorAndReturn(err, "Error creating garden k8s client", composed.StopAndForget, nil) + return composed.LogErrorAndReturn(err, "Error creating garden k8s client", composed.StopAndForget, ctx) } state.gardenK8sClient = k8sClient diff --git a/components/kcp/pkg/kcp/scope/createScopeAws.go b/components/kcp/pkg/kcp/scope/createScopeAws.go index 66d92b70e..dbfbf72a5 100644 --- a/components/kcp/pkg/kcp/scope/createScopeAws.go +++ b/components/kcp/pkg/kcp/scope/createScopeAws.go @@ -26,7 +26,7 @@ func createScopeAws(ctx context.Context, st composed.State) (error, context.Cont fmt.Errorf("error creating aws scope: %w", err), "Error creating AWS scope", composed.StopAndForget, - nil) + ctx) } callerIdentity, err := stsClient.GetCallerIdentity(ctx) if err != nil { @@ -34,13 +34,13 @@ func createScopeAws(ctx context.Context, st composed.State) (error, context.Cont fmt.Errorf("error getting caller identity: %w", err), "Error creating AWS scope", composed.StopWithRequeue, - nil) + ctx) } infra := &awsgardener.InfrastructureConfig{} err = json.Unmarshal(state.shoot.Spec.Provider.InfrastructureConfig.Raw, infra) if err != nil { - return composed.LogErrorAndReturn(err, "Error unmarshalling InfrastructureConfig", composed.StopAndForget, nil) + return composed.LogErrorAndReturn(err, "Error unmarshalling InfrastructureConfig", composed.StopAndForget, ctx) } scope := &cloudcontrolv1beta1.Scope{ diff --git a/components/kcp/pkg/kcp/scope/ensureScopeCommonFields.go b/components/kcp/pkg/kcp/scope/ensureScopeCommonFields.go index 9843051f3..00c8d2dbc 100644 --- a/components/kcp/pkg/kcp/scope/ensureScopeCommonFields.go +++ b/components/kcp/pkg/kcp/scope/ensureScopeCommonFields.go @@ -13,7 +13,7 @@ func ensureScopeCommonFields(ctx context.Context, st composed.State) (error, con // set name state.ObjAsScope().Name = state.kyma.GetName() // same name as Kyma CR - state.ObjAsScope().Namespace = state.Obj().GetNamespace() + state.ObjAsScope().Namespace = state.kyma.GetNamespace() // set finalizer controllerutil.AddFinalizer(state.ObjAsScope(), cloudcontrolv1beta1.FinalizerName) diff --git a/components/kcp/pkg/kcp/scope/ignoreScopeObj.go b/components/kcp/pkg/kcp/scope/ignoreScopeObj.go new file mode 100644 index 000000000..40a35756b --- /dev/null +++ b/components/kcp/pkg/kcp/scope/ignoreScopeObj.go @@ -0,0 +1,13 @@ +package scope + +import ( + "context" + "github.com/kyma-project/cloud-manager/components/lib/composed" +) + +func ignoreScopeObj(ctx context.Context, st composed.State) (error, context.Context) { + if st.Obj().GetObjectKind().GroupVersionKind().Kind == "Scope" { + return composed.StopAndForget, nil + } + return nil, nil +} diff --git a/components/kcp/pkg/kcp/scope/loadGardenerCredentials.go b/components/kcp/pkg/kcp/scope/loadGardenerCredentials.go index c7d38e553..0fb967af8 100644 --- a/components/kcp/pkg/kcp/scope/loadGardenerCredentials.go +++ b/components/kcp/pkg/kcp/scope/loadGardenerCredentials.go @@ -9,13 +9,13 @@ import ( func loadGardenerCredentials(ctx context.Context, st composed.State) (error, context.Context) { logger := composed.LoggerFromCtx(ctx) - state := st.(State) + state := st.(*State) bindingName := *state.shoot.Spec.SecretBindingName secretBinding, err := state.gardenerClient.SecretBindings(state.shootNamespace).Get(ctx, bindingName, metav1.GetOptions{}) if err != nil { - return composed.LogErrorAndReturn(err, "Error getting shoot secret binding", composed.StopWithRequeue, nil) + return composed.LogErrorAndReturn(err, "Error getting shoot secret binding", composed.StopWithRequeue, ctx) } state.provider = cloudcontrolv1beta1.ProviderType(secretBinding.Provider.Type) @@ -23,7 +23,7 @@ func loadGardenerCredentials(ctx context.Context, st composed.State) (error, con secret, err := state.gardenK8sClient.CoreV1().Secrets(secretBinding.SecretRef.Namespace). Get(ctx, secretBinding.SecretRef.Name, metav1.GetOptions{}) if err != nil { - return composed.LogErrorAndReturn(err, "Error getting shoot secret", composed.StopWithRequeue, nil) + return composed.LogErrorAndReturn(err, "Error getting shoot secret", composed.StopWithRequeue, ctx) } for k, v := range secret.Data { diff --git a/components/kcp/pkg/kcp/scope/loadKyma.go b/components/kcp/pkg/kcp/scope/loadKyma.go index 0968f96ee..5da5b2b8c 100644 --- a/components/kcp/pkg/kcp/scope/loadKyma.go +++ b/components/kcp/pkg/kcp/scope/loadKyma.go @@ -5,7 +5,6 @@ import ( "github.com/kyma-project/cloud-manager/components/kcp/pkg/util" "github.com/kyma-project/cloud-manager/components/lib/composed" apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/types" ) func loadKyma(ctx context.Context, st composed.State) (error, context.Context) { @@ -13,10 +12,7 @@ func loadKyma(ctx context.Context, st composed.State) (error, context.Context) { state := st.(*State) kymaUnstructured := util.NewKymaUnstructured() - err := state.Cluster().K8sClient().Get(ctx, types.NamespacedName{ - Name: state.Obj().GetName(), - Namespace: state.Obj().GetNamespace(), - }, kymaUnstructured) + err := state.Cluster().K8sClient().Get(ctx, state.Name(), kymaUnstructured) if apierrors.IsNotFound(err) { logger.Info("Kyma CR does not exist") @@ -24,7 +20,7 @@ func loadKyma(ctx context.Context, st composed.State) (error, context.Context) { } if err != nil { - return composed.LogErrorAndReturn(err, "Error loading Kyma CR", composed.StopWithRequeue, nil) + return composed.LogErrorAndReturn(err, "Error loading Kyma CR", composed.StopWithRequeue, ctx) } state.kyma = kymaUnstructured diff --git a/components/kcp/pkg/kcp/scope/loadScopeObj.go b/components/kcp/pkg/kcp/scope/loadScopeObj.go index e0aba8e85..6e42f2c61 100644 --- a/components/kcp/pkg/kcp/scope/loadScopeObj.go +++ b/components/kcp/pkg/kcp/scope/loadScopeObj.go @@ -2,6 +2,7 @@ package scope import ( "context" + cloudcontrolv1beta1 "github.com/kyma-project/cloud-manager/components/kcp/api/cloud-control/v1beta1" "github.com/kyma-project/cloud-manager/components/lib/composed" apierrors "k8s.io/apimachinery/pkg/api/errors" ) @@ -10,12 +11,15 @@ func loadScopeObj(ctx context.Context, state composed.State) (error, context.Con logger := composed.LoggerFromCtx(ctx) err := state.LoadObj(ctx) if apierrors.IsNotFound(err) { + list := &cloudcontrolv1beta1.ScopeList{} + err = state.Cluster().K8sClient().List(ctx, list) + logger.Info("Scope object does not exist") // continue to create one return nil, nil } if err != nil { - return composed.LogErrorAndReturn(err, "Error loading Scope object", composed.StopWithRequeue, nil) + return composed.LogErrorAndReturn(err, "Error loading Scope object", composed.StopWithRequeue, ctx) } return composed.StopAndForget, nil diff --git a/components/kcp/pkg/kcp/scope/loadShoot.go b/components/kcp/pkg/kcp/scope/loadShoot.go index c162fe97b..adce24b6a 100644 --- a/components/kcp/pkg/kcp/scope/loadShoot.go +++ b/components/kcp/pkg/kcp/scope/loadShoot.go @@ -8,11 +8,15 @@ import ( func loadShoot(ctx context.Context, st composed.State) (error, context.Context) { logger := composed.LoggerFromCtx(ctx) - state := st.(State) + state := st.(*State) shoot, err := state.gardenerClient.Shoots(state.shootNamespace).Get(ctx, state.shootName, metav1.GetOptions{}) if err != nil { - return composed.LogErrorAndReturn(err, "Error getting Shoot", composed.StopWithRequeue, nil) + ctx = composed.LoggerIntoCtx(ctx, logger.WithValues( + "shootNamespace", state.shootNamespace, + "shootName", state.shootName, + )) + return composed.LogErrorAndReturn(err, "Error getting Shoot", composed.StopWithRequeue, ctx) } state.shoot = shoot diff --git a/components/kcp/pkg/kcp/scope/reconciler.go b/components/kcp/pkg/kcp/scope/reconciler.go index cb63f5582..768edc680 100644 --- a/components/kcp/pkg/kcp/scope/reconciler.go +++ b/components/kcp/pkg/kcp/scope/reconciler.go @@ -2,8 +2,12 @@ package scope import ( "context" + scopeclient "github.com/kyma-project/cloud-manager/components/kcp/pkg/kcp/scope/client" + awsclient "github.com/kyma-project/cloud-manager/components/kcp/pkg/provider/aws/client" + skrruntime "github.com/kyma-project/cloud-manager/components/kcp/pkg/skr/runtime" "github.com/kyma-project/cloud-manager/components/lib/composed" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" ) @@ -11,6 +15,18 @@ type ScopeReconciler interface { reconcile.Reconciler } +func New( + mgr manager.Manager, + awsStsClientProvider awsclient.GardenClientProvider[scopeclient.AwsStsClient], + activeSkrCollection skrruntime.ActiveSkrCollection, +) ScopeReconciler { + return NewScopeReconciler(NewStateFactory( + composed.NewStateFactory(composed.NewStateClusterFromManager(mgr)), + awsStsClientProvider, + activeSkrCollection, + )) +} + func NewScopeReconciler(stateFactory StateFactory) ScopeReconciler { return &scopeReconciler{ stateFactory: stateFactory, @@ -56,6 +72,7 @@ func (r *scopeReconciler) newAction() composed.Action { // scope does not exist createGardenerClient, + findShootName, loadShoot, loadGardenerCredentials, createScope, diff --git a/components/kcp/pkg/kcp/scope/saveScope.go b/components/kcp/pkg/kcp/scope/saveScope.go index cf9a26015..b5e9e3d26 100644 --- a/components/kcp/pkg/kcp/scope/saveScope.go +++ b/components/kcp/pkg/kcp/scope/saveScope.go @@ -6,11 +6,11 @@ import ( ) func saveScope(ctx context.Context, st composed.State) (error, context.Context) { - state := st.(State) + state := st.(*State) err := state.Cluster().K8sClient().Create(ctx, state.ObjAsScope()) if err != nil { - return composed.LogErrorAndReturn(err, "Error creating scope", composed.StopWithRequeue, nil) + return composed.LogErrorAndReturn(err, "Error creating scope", composed.StopWithRequeue, ctx) } return nil, nil diff --git a/components/kcp/pkg/kcp/scope/state.go b/components/kcp/pkg/kcp/scope/state.go index f869238be..c74ea1536 100644 --- a/components/kcp/pkg/kcp/scope/state.go +++ b/components/kcp/pkg/kcp/scope/state.go @@ -4,7 +4,6 @@ import ( gardenerTypes "github.com/gardener/gardener/pkg/apis/core/v1beta1" gardenerClient "github.com/gardener/gardener/pkg/client/core/clientset/versioned/typed/core/v1beta1" cloudcontrolv1beta1 "github.com/kyma-project/cloud-manager/components/kcp/api/cloud-control/v1beta1" - "github.com/kyma-project/cloud-manager/components/kcp/pkg/common/abstractions" scopeclient "github.com/kyma-project/cloud-manager/components/kcp/pkg/kcp/scope/client" awsClient "github.com/kyma-project/cloud-manager/components/kcp/pkg/provider/aws/client" skrruntime "github.com/kyma-project/cloud-manager/components/kcp/pkg/skr/runtime" @@ -21,20 +20,20 @@ type StateFactory interface { func NewStateFactory( baseStateFactory composed.StateFactory, - fileReader abstractions.FileReader, awsStsClientProvider awsClient.GardenClientProvider[scopeclient.AwsStsClient], + activeSkrCollection skrruntime.ActiveSkrCollection, ) StateFactory { return &stateFactory{ baseStateFactory: baseStateFactory, - fileReader: fileReader, awsStsClientProvider: awsStsClientProvider, + activeSkrCollection: activeSkrCollection, } } type stateFactory struct { baseStateFactory composed.StateFactory - fileReader abstractions.FileReader awsStsClientProvider awsClient.GardenClientProvider[scopeclient.AwsStsClient] + activeSkrCollection skrruntime.ActiveSkrCollection } func (f *stateFactory) NewState(req ctrl.Request) *State { @@ -42,18 +41,22 @@ func (f *stateFactory) NewState(req ctrl.Request) *State { return newState( baseState, - f.fileReader, f.awsStsClientProvider, + f.activeSkrCollection, ) } // ===================================================================== -func newState(baseState composed.State, fileReader abstractions.FileReader, awsStsClientProvider awsClient.GardenClientProvider[scopeclient.AwsStsClient]) *State { +func newState( + baseState composed.State, + awsStsClientProvider awsClient.GardenClientProvider[scopeclient.AwsStsClient], + activeSkrCollection skrruntime.ActiveSkrCollection, +) *State { return &State{ State: baseState, - fileReader: fileReader, awsStsClientProvider: awsStsClientProvider, + activeSkrCollection: activeSkrCollection, credentialData: map[string]string{}, } } @@ -61,8 +64,6 @@ func newState(baseState composed.State, fileReader abstractions.FileReader, awsS type State struct { composed.State - fileReader abstractions.FileReader - kyma *unstructured.Unstructured moduleState util.KymaModuleState activeSkrCollection skrruntime.ActiveSkrCollection diff --git a/components/kcp/pkg/nfsinstance/reconciler.go b/components/kcp/pkg/nfsinstance/reconciler.go index fd682f71d..dc2713f3d 100644 --- a/components/kcp/pkg/nfsinstance/reconciler.go +++ b/components/kcp/pkg/nfsinstance/reconciler.go @@ -2,6 +2,7 @@ package nfsinstance import ( "context" + "sigs.k8s.io/controller-runtime/pkg/reconcile" cloudresourcesv1beta1 "github.com/kyma-project/cloud-manager/components/kcp/api/cloud-control/v1beta1" "github.com/kyma-project/cloud-manager/components/kcp/pkg/common/actions/focal" @@ -13,7 +14,11 @@ import ( ctrl "sigs.k8s.io/controller-runtime" ) -type NfsInstanceReconciler struct { +type NfsInstanceReconciler interface { + reconcile.Reconciler +} + +type nfsInstanceReconciler struct { composedStateFactory composed.StateFactory focalStateFactory focal.StateFactory @@ -28,8 +33,8 @@ func NewNfsInstanceReconciler( awsStateFactory awsnfsinstance.StateFactory, azureStateFactory azurenfsinstance.StateFactory, gcpStateFactory gcpnfsinstance.StateFactory, -) *NfsInstanceReconciler { - return &NfsInstanceReconciler{ +) NfsInstanceReconciler { + return &nfsInstanceReconciler{ composedStateFactory: composedStateFactory, focalStateFactory: focalStateFactory, awsStateFactory: awsStateFactory, @@ -38,14 +43,14 @@ func NewNfsInstanceReconciler( } } -func (r *NfsInstanceReconciler) Run(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { +func (r *nfsInstanceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { state := r.newFocalState(req.NamespacedName) action := r.newAction() return composed.Handle(action(ctx, state)) } -func (r *NfsInstanceReconciler) newAction() composed.Action { +func (r *nfsInstanceReconciler) newAction() composed.Action { return composed.ComposeActions( "main", focal.New(), @@ -67,7 +72,7 @@ func (r *NfsInstanceReconciler) newAction() composed.Action { ) } -func (r *NfsInstanceReconciler) newFocalState(name types.NamespacedName) focal.State { +func (r *nfsInstanceReconciler) newFocalState(name types.NamespacedName) focal.State { return r.focalStateFactory.NewState( r.composedStateFactory.NewState(name, &cloudresourcesv1beta1.NfsInstance{}), ) diff --git a/components/kcp/pkg/provider/aws/gardener/infrastructure.go b/components/kcp/pkg/provider/aws/gardener/infrastructure.go index 6e476f36b..b29597c75 100644 --- a/components/kcp/pkg/provider/aws/gardener/infrastructure.go +++ b/components/kcp/pkg/provider/aws/gardener/infrastructure.go @@ -1,6 +1,9 @@ package gardener -import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" +) /** * To avoid as reference copied from @@ -12,6 +15,13 @@ type InfrastructureConfig struct { Networks Networks `json:"networks"` } +func (i *InfrastructureConfig) DeepCopyObject() runtime.Object { + return &InfrastructureConfig{ + TypeMeta: i.TypeMeta, + Networks: i.Networks, + } +} + type Networks struct { // VPC indicates whether to use an existing VPC or create a new one. VPC VPC `json:"vpc"` diff --git a/components/kcp/pkg/skr/runtime/alias.go b/components/kcp/pkg/skr/runtime/alias.go index 2275e1167..cebeee645 100644 --- a/components/kcp/pkg/skr/runtime/alias.go +++ b/components/kcp/pkg/skr/runtime/alias.go @@ -7,6 +7,8 @@ import ( var NewLooper = looper.New +type SkrLooper = looper.SkrLooper + var NewRegistry = registry.New var NewRunner = looper.NewSkrRunner diff --git a/components/kcp/pkg/skr/runtime/registry/builder.go b/components/kcp/pkg/skr/runtime/registry/builder.go index 80ec28f63..179f01a97 100644 --- a/components/kcp/pkg/skr/runtime/registry/builder.go +++ b/components/kcp/pkg/skr/runtime/registry/builder.go @@ -64,7 +64,7 @@ func (b *skrBuilder) Complete() error { return errors.New("the For(...) method must be called to define reconciled object") } - gvk, err := util.GetObjGvk(b.registry.scheme, b.forInput.object) + gvk, err := util.GetObjGvk(b.registry.skrScheme, b.forInput.object) if err != nil { return err } @@ -78,7 +78,7 @@ func (b *skrBuilder) Complete() error { reconcilerFactory: b.factory, } - gvk, err = util.GetObjGvk(b.registry.scheme, b.forInput.object) + gvk, err = util.GetObjGvk(b.registry.skrScheme, b.forInput.object) if err != nil { return err } @@ -92,7 +92,7 @@ func (b *skrBuilder) Complete() error { }) for _, w := range b.watchesInput { - gvk, err := util.GetObjGvk(b.registry.scheme, w.object) + gvk, err := util.GetObjGvk(b.registry.skrScheme, w.object) if err != nil { return err } diff --git a/components/kcp/pkg/skr/runtime/registry/registry.go b/components/kcp/pkg/skr/runtime/registry/registry.go index 81122a150..898532a98 100644 --- a/components/kcp/pkg/skr/runtime/registry/registry.go +++ b/components/kcp/pkg/skr/runtime/registry/registry.go @@ -54,13 +54,14 @@ type Descriptor struct { } type SkrRegistry interface { + Len() int GetDescriptors(mngr manager.SkrManager) DescriptorList Register() Builder } -func New(scheme *runtime.Scheme) SkrRegistry { +func New(skrScheme *runtime.Scheme) SkrRegistry { return &skrRegistry{ - scheme: scheme, + skrScheme: skrScheme, } } @@ -79,8 +80,12 @@ type registryItemWatch struct { } type skrRegistry struct { - scheme *runtime.Scheme - items []*registryItem + skrScheme *runtime.Scheme + items []*registryItem +} + +func (r *skrRegistry) Len() int { + return len(r.items) } func (r *skrRegistry) GetDescriptors(mngr manager.SkrManager) DescriptorList { diff --git a/components/kcp/pkg/testinfra/crd.go b/components/kcp/pkg/testinfra/crd.go index b00fda0e0..86a64442e 100644 --- a/components/kcp/pkg/testinfra/crd.go +++ b/components/kcp/pkg/testinfra/crd.go @@ -1,10 +1,10 @@ package testinfra import ( + "errors" "fmt" "io" "os" - "os/exec" "path" "path/filepath" "strings" @@ -14,104 +14,109 @@ import ( var crdsInitialized = false var ( - dirSkr string - dirKcp string - dirGarden string - crdMutex sync.Mutex ) -func initCrds() error { +func initCrds(projectRoot string) (dirSkr, dirKcp, dirGarden string, err error) { crdMutex.Lock() defer crdMutex.Unlock() if crdsInitialized { - return nil - } - - localBin := os.Getenv("LOCALBIN") - if len(localBin) == 0 { - localBin = filepath.Join("..", "..", "bin") + err = errors.New("the CRDs are already initialized") + return } - dirSkr = path.Join(localBin, "cloud-manager", "skr") - dirKcp = path.Join(localBin, "cloud-manager", "kcp") - dirGarden = path.Join(localBin, "cloud-manager", "garden") + dirSkr = path.Join(projectRoot, "bin", "cloud-manager", "skr") + dirKcp = path.Join(projectRoot, "bin", "cloud-manager", "kcp") + dirGarden = path.Join(projectRoot, "bin", "cloud-manager", "garden") // recreate destination directories - if err := createDir(dirSkr); err != nil { - return err + if err = createDir(dirSkr); err != nil { + return } - if err := createDir(dirKcp); err != nil { - return err + if err = createDir(dirKcp); err != nil { + return } - if err := createDir(dirGarden); err != nil { - return err + if err = createDir(dirGarden); err != nil { + return } - // copy CRDs to destination dirs - configCrdBases := filepath.Join("..", "..", "config", "crd", "bases") - list, err := os.ReadDir(configCrdBases) - if err != nil { - return fmt.Errorf("error reading files in config/crd/bases dir: %w", err) - } + var list []os.DirEntry - prefixMap := map[string]string{ - "cloud-control.": dirKcp, - "cloud-resources.": dirSkr, - } - for _, x := range list { - if x.IsDir() { - continue + // copy cloud-manager CRDs to destination dirs + { + configCrdBases := filepath.Join(projectRoot, "config", "crd", "bases") + list, err = os.ReadDir(configCrdBases) + if err != nil { + err = fmt.Errorf("error reading files in config/crd/bases dir: %w", err) + return } - for prefix, dir := range prefixMap { - if strings.HasPrefix(x.Name(), prefix) { - err := copyFile( - filepath.Join(configCrdBases, x.Name()), - filepath.Join(dir, x.Name()), - ) - if err != nil { - return err + + prefixMap := map[string]string{ + "cloud-control.": dirKcp, + "cloud-resources.": dirSkr, + } + for _, x := range list { + if x.IsDir() { + continue + } + for prefix, dir := range prefixMap { + if strings.HasPrefix(x.Name(), prefix) { + err = copyFile( + filepath.Join(configCrdBases, x.Name()), + filepath.Join(dir, x.Name()), + ) + if err != nil { + return + } + break } - break } } } - // generate all gardener CRDs - cmd := exec.Command( - filepath.Join(localBin, "controller-gen"), - "crd:allowDangerousTypes=true", - fmt.Sprintf(`paths="%s/pkg/mod/github.com/gardener/gardener@v1.85.0/pkg/apis/core/v1beta1"`, os.Getenv("GOPATH")), - fmt.Sprintf("output:crd:artifacts:config=%s", dirGarden), - ) - err = cmd.Run() - if err != nil { - return err + // copy gardener CRDs + { + gardenerCrdsDir := filepath.Join(projectRoot, "config", "crd", "gardener") + list, err = os.ReadDir(gardenerCrdsDir) + if err != nil { + err = fmt.Errorf("error listing gardener crds: %w", err) + return + } + for _, f := range list { + err = copyFile( + filepath.Join(gardenerCrdsDir, f.Name()), + filepath.Join(dirGarden, f.Name()), + ) + if err != nil { + err = fmt.Errorf("error copying gardener crd %s: %w", f.Name(), err) + return + } + } } - // delete all garedener CRDs except few we want to keep - gardenFilesToKeep := map[string]struct{}{ - "core.gardener.cloud_shoots.yaml": {}, - "core.gardener.cloud_secretbindings.yaml": {}, - } - list, err = os.ReadDir(dirGarden) - if err != nil { - return err - } - for _, f := range list { - _, keep := gardenFilesToKeep[f.Name()] - if keep { - continue - } - err = os.Remove(filepath.Join(dirGarden, f.Name())) + // copy operator CRDs + { + operatorCrdsDir := filepath.Join(projectRoot, "config", "crd", "operator") + list, err = os.ReadDir(operatorCrdsDir) if err != nil { - return err + err = fmt.Errorf("error listing operator crds: %w", err) + return + } + for _, f := range list { + err = copyFile( + filepath.Join(operatorCrdsDir, f.Name()), + filepath.Join(dirKcp, f.Name()), + ) + if err != nil { + err = fmt.Errorf("error copying operator crd %s: %w", f.Name(), err) + return + } } } crdsInitialized = true - return nil + return } func createDir(dir string) error { diff --git a/components/kcp/pkg/testinfra/dsl.go b/components/kcp/pkg/testinfra/dsl.go new file mode 100644 index 000000000..d249bad05 --- /dev/null +++ b/components/kcp/pkg/testinfra/dsl.go @@ -0,0 +1,279 @@ +package testinfra + +import ( + "context" + "fmt" + gardenerTypes "github.com/gardener/gardener/pkg/apis/core/v1beta1" + awsgardener "github.com/kyma-project/cloud-manager/components/kcp/pkg/provider/aws/gardener" + "github.com/kyma-project/cloud-manager/components/kcp/pkg/util" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +var _ InfraDSL = &infraDSL{} + +type infraDSL struct { + i Infra +} + +func (dsl *infraDSL) GivenKymaCRExists(name string) error { + kymaCR := util.NewKymaUnstructured() + kymaCR.SetLabels(map[string]string{ + "kyma-project.io/shoot-name": name, + }) + kymaCR.SetName(name) + kymaCR.SetNamespace(dsl.i.KCP().Namespace()) + + err := dsl.i.KCP().Client().Get(dsl.i.Ctx(), client.ObjectKeyFromObject(kymaCR), kymaCR) + if err == nil { + // already exist + return nil + } + if client.IgnoreNotFound(err) != nil { + // some error + return err + } + err = dsl.i.KCP().Client().Create(dsl.i.Ctx(), kymaCR) + if err != nil { + return err + } + + // Kubeconfig secret + { + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: dsl.i.KCP().Namespace(), + Name: fmt.Sprintf("kubeconfig-%s", name), + }, + } + err := dsl.i.KCP().Client().Get(dsl.i.Ctx(), client.ObjectKeyFromObject(secret), secret) + if client.IgnoreNotFound(err) != nil { + return err + } + if apierrors.IsNotFound(err) { + b, err := kubeconfigToBytes(restConfigToKubeconfig(dsl.i.SKR().Cfg())) + if err != nil { + return fmt.Errorf("error getting SKR kubeconfig bytes: %w", err) + } + secret.Data = map[string][]byte{ + "config": b, + } + + err = dsl.i.KCP().Client().Create(dsl.i.Ctx(), secret) + if client.IgnoreAlreadyExists(err) != nil { + return fmt.Errorf("error creating SKR secret: %w", err) + } + } + } + + return nil +} + +func (dsl *infraDSL) GivenGardenShootAwsExists(name string) error { + // Gardener-credentials secret + { + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: dsl.i.KCP().Namespace(), + Name: "gardener-credentials", + }, + } + err := dsl.i.KCP().Client().Get(dsl.i.Ctx(), client.ObjectKeyFromObject(secret), secret) + if client.IgnoreNotFound(err) != nil { + return err + } + if apierrors.IsNotFound(err) { + b, err := kubeconfigToBytes(restConfigToKubeconfig(dsl.i.Garden().Cfg())) + if err != nil { + return fmt.Errorf("error getting garden kubeconfig bytes: %w", err) + } + secret.Data = map[string][]byte{ + "kubeconfig": b, + } + + err = dsl.i.KCP().Client().Create(dsl.i.Ctx(), secret) + if client.IgnoreAlreadyExists(err) != nil { + return fmt.Errorf("error creating gardener-credentials secret: %w", err) + } + } + } + + // Shoot + { + shoot := &gardenerTypes.Shoot{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: dsl.i.Garden().Namespace(), + Name: name, + }, + Spec: gardenerTypes.ShootSpec{ + Region: "us-central1", + Networking: &gardenerTypes.Networking{ + IPFamilies: []gardenerTypes.IPFamily{gardenerTypes.IPFamilyIPv4}, + Nodes: pointer.String("10.180.0.0/16"), + Pods: pointer.String("100.64.0.0/12"), + Services: pointer.String("100.104.0.0/13"), + }, + Provider: gardenerTypes.Provider{ + Type: "aws", + InfrastructureConfig: &runtime.RawExtension{ + Object: &awsgardener.InfrastructureConfig{ + TypeMeta: metav1.TypeMeta{ + Kind: "InfrastructureConfig", + APIVersion: "aws.provider.extensions.gardener.cloud/v1alpha1", + }, + Networks: awsgardener.Networks{ + VPC: awsgardener.VPC{ + CIDR: pointer.String("10.180.0.0/16"), + }, + Zones: []awsgardener.Zone{ + { + Name: "eu-west-1a", + Internal: "10.180.48.0/20", + Public: "10.180.32.0/20", + Workers: "10.180.0.0/19", + }, + { + Name: "eu-west-1b", + Internal: "10.180.112.0/20", + Public: "10.180.96.0/20", + Workers: "10.180.64.0/19", + }, + { + Name: "eu-west-1c", + Internal: "10.180.176.0/20", + Public: "10.180.160.0/20", + Workers: "10.180.128.0/19", + }, + }, + }, + }, + }, + }, + SecretBindingName: pointer.String(name), + }, + } + err := dsl.i.Garden().Client().Create(dsl.i.Ctx(), shoot) + if err != nil { + return err + } + } + + // SecretBinding + { + secretBinding := &gardenerTypes.SecretBinding{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: dsl.i.Garden().Namespace(), + Name: name, + }, + Provider: &gardenerTypes.SecretBindingProvider{ + Type: "aws", + }, + SecretRef: corev1.SecretReference{ + Name: name, + Namespace: dsl.i.Garden().Namespace(), + }, + } + err := dsl.i.Garden().Client().Create(dsl.i.Ctx(), secretBinding) + if err != nil { + return err + } + } + + // Secret + { + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: dsl.i.Garden().Namespace(), + Name: name, + }, + StringData: map[string]string{ + "accessKeyID": "accessKeyID", + "secretAccessKey": "secretAccessKey", + }, + } + err := dsl.i.Garden().Client().Create(dsl.i.Ctx(), secret) + if err != nil { + return err + } + } + + return nil +} + +func (dsl *infraDSL) WhenKymaModuleStateUpdates(kymaName string, state util.KymaModuleState) error { + kymaCR := util.NewKymaUnstructured() + err := dsl.i.KCP().Client().Get(dsl.i.Ctx(), types.NamespacedName{ + Namespace: dsl.i.KCP().Namespace(), + Name: kymaName, + }, kymaCR) + if err != nil { + return err + } + + err = util.SetKymaModuleState(kymaCR, "cloud-manager", state) + if err != nil { + return err + } + + err = dsl.i.KCP().Client().Status().Update(dsl.i.Ctx(), kymaCR) + if err != nil { + return err + } + + return nil +} + +// ======================= + +var _ ClusterDSL = &clusterDSL{} + +type clusterDSL struct { + ci ClusterInfo + ctx func() context.Context +} + +func (dsl *clusterDSL) GivenSecretExists(name string, data map[string][]byte) error { + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: dsl.ci.Namespace(), + Name: name, + }, + } + err := dsl.ci.Client().Get(dsl.ctx(), client.ObjectKeyFromObject(secret), secret) + if client.IgnoreNotFound(err) != nil { + return err + } + secret.Data = data + if apierrors.IsNotFound(err) { + err = dsl.ci.Client().Create(dsl.ctx(), secret) + } else { + err = dsl.ci.Client().Update(dsl.ctx(), secret) + } + return err + +} + +func (dsl *clusterDSL) GivenNamespaceExists(name string) error { + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + } + err := dsl.ci.Client().Get(dsl.ctx(), client.ObjectKeyFromObject(ns), ns) + if err == nil { + // already exists + return nil + } + if client.IgnoreNotFound(err) != nil { + // some error + return err + } + err = dsl.ci.Client().Create(dsl.ctx(), ns) + return err + +} diff --git a/components/kcp/pkg/testinfra/env.go b/components/kcp/pkg/testinfra/env.go new file mode 100644 index 000000000..3f285f18e --- /dev/null +++ b/components/kcp/pkg/testinfra/env.go @@ -0,0 +1,87 @@ +package testinfra + +import ( + "context" + awsmock "github.com/kyma-project/cloud-manager/components/kcp/pkg/provider/aws/mock" + skrruntime "github.com/kyma-project/cloud-manager/components/kcp/pkg/skr/runtime" + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/manager" + "time" +) + +var _ InfraEnv = &infraEnv{} + +type infraEnv struct { + kcpManager manager.Manager + registry skrruntime.SkrRegistry + looper skrruntime.SkrLooper + awsMock awsmock.Server + + ctx context.Context + cancel context.CancelFunc +} + +func (ie *infraEnv) KcpManager() manager.Manager { + return ie.kcpManager +} + +func (ie *infraEnv) Registry() skrruntime.SkrRegistry { + return ie.registry +} + +func (ie *infraEnv) Looper() skrruntime.SkrLooper { + return ie.looper +} + +func (ie *infraEnv) AwsMock() awsmock.Server { + return ie.awsMock +} + +func (ie *infraEnv) StartControllers(ctx context.Context) { + ginkgo.By("Starting controllers") + if ctx == nil { + ctx = context.Background() + } + ie.ctx, ie.cancel = context.WithCancel(ctx) + go func() { + defer ginkgo.GinkgoRecover() + err := ie.kcpManager.Start(ie.ctx) + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "failed to run kcp manager") + }() + time.Sleep(10 * time.Millisecond) +} + +func (ie *infraEnv) Ctx() context.Context { + if ie.ctx == nil { + return context.Background() + } + return ie.ctx +} + +func (ie *infraEnv) stopControllers() { + if ie.cancel != nil { + ginkgo.By("Stopping controllers") + ie.cancel() + } +} + +// ========================================= + +var _ ClusterEnv = &clusterEnv{} + +type clusterEnv struct { + namespace string +} + +func (ce *clusterEnv) Namespace() string { + return ce.namespace +} + +func (ce *clusterEnv) ObjKey(name string) types.NamespacedName { + return types.NamespacedName{ + Namespace: ce.namespace, + Name: name, + } +} diff --git a/components/kcp/pkg/testinfra/infra.go b/components/kcp/pkg/testinfra/infra.go new file mode 100644 index 000000000..8f48fe01d --- /dev/null +++ b/components/kcp/pkg/testinfra/infra.go @@ -0,0 +1,72 @@ +package testinfra + +import ( + "fmt" + "github.com/onsi/ginkgo/v2" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" +) + +var _ Infra = &infra{} + +type infra struct { + InfraEnv + InfraDSL + + clusters map[ClusterType]*clusterInfo +} + +func (i *infra) KCP() ClusterInfo { + return i.clusters[ClusterTypeKcp] +} + +func (i *infra) SKR() ClusterInfo { + return i.clusters[ClusterTypeSkr] +} + +func (i *infra) Garden() ClusterInfo { + return i.clusters[ClusterTypeGarden] +} + +func (i *infra) Stop() error { + i.stopControllers() + + var lastErr error + for name, cluster := range i.clusters { + ginkgo.By(fmt.Sprintf("Stopping cluster %s", name)) + if err := cluster.env.Stop(); err != nil { + lastErr = err + } + } + + return lastErr +} + +// ======================= + +var _ ClusterInfo = &clusterInfo{} + +type clusterInfo struct { + ClusterEnv + ClusterDSL + + crdDirs []string + env *envtest.Environment + cfg *rest.Config + scheme *runtime.Scheme + client client.Client +} + +func (c *clusterInfo) Scheme() *runtime.Scheme { + return c.scheme +} + +func (c *clusterInfo) Client() client.Client { + return c.client +} + +func (c *clusterInfo) Cfg() *rest.Config { + return c.cfg +} diff --git a/components/kcp/pkg/testinfra/restConfigToKubeconfig.go b/components/kcp/pkg/testinfra/restConfigToKubeconfig.go new file mode 100644 index 000000000..1fe79297d --- /dev/null +++ b/components/kcp/pkg/testinfra/restConfigToKubeconfig.go @@ -0,0 +1,42 @@ +package testinfra + +import ( + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" +) + +func restConfigToKubeconfig(restConfig *rest.Config) *clientcmdapi.Config { + clusters := make(map[string]*clientcmdapi.Cluster) + clusters["default-cluster"] = &clientcmdapi.Cluster{ + Server: restConfig.Host, + CertificateAuthorityData: restConfig.CAData, + } + + contexts := make(map[string]*clientcmdapi.Context) + contexts["default-context"] = &clientcmdapi.Context{ + Cluster: "default-cluster", + AuthInfo: "default-auth", + } + + authinfos := make(map[string]*clientcmdapi.AuthInfo) + authinfos["default-auth"] = &clientcmdapi.AuthInfo{ + ClientCertificateData: restConfig.CertData, + ClientKeyData: restConfig.KeyData, + } + + clientConfig := &clientcmdapi.Config{ + Kind: "Config", + APIVersion: "v1", + Clusters: clusters, + Contexts: contexts, + CurrentContext: "default-context", + AuthInfos: authinfos, + } + + return clientConfig +} + +func kubeconfigToBytes(clientConfig *clientcmdapi.Config) ([]byte, error) { + return clientcmd.Write(*clientConfig) +} diff --git a/components/kcp/pkg/testinfra/run.go b/components/kcp/pkg/testinfra/run.go new file mode 100644 index 000000000..515291a10 --- /dev/null +++ b/components/kcp/pkg/testinfra/run.go @@ -0,0 +1,138 @@ +package testinfra + +import ( + "errors" + "fmt" + awsmock "github.com/kyma-project/cloud-manager/components/kcp/pkg/provider/aws/mock" + skrruntime "github.com/kyma-project/cloud-manager/components/kcp/pkg/skr/runtime" + "github.com/onsi/ginkgo/v2" + "k8s.io/client-go/rest" + "os" + "path/filepath" + goruntime "runtime" + ctrl "sigs.k8s.io/controller-runtime" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" +) + +func Start() (Infra, error) { + projectRoot := os.Getenv("PROJECTROOT") + if len(projectRoot) == 0 { + return nil, errors.New("the env var PROJECTROOT must be specified and point to the dir where Makefile is") + } + envtestK8sVersion := os.Getenv("ENVTEST_K8S_VERSION") + if len(envtestK8sVersion) == 0 { + envtestK8sVersion = "1.28.0" + } + + ginkgo.By("Preparing CRDs") + + dirSkr, dirKcp, dirGarden, err := initCrds(projectRoot) + if err != nil { + return nil, fmt.Errorf("error initializing CRDs: %w", err) + } + + configDir := filepath.Join(projectRoot, "bin", "cloud-manager", "config") + if err := os.MkdirAll(configDir, 0777); err != nil { + return nil, fmt.Errorf("error creating config dir: %w", err) + } + + infra := &infra{ + clusters: map[ClusterType]*clusterInfo{ + ClusterTypeKcp: &clusterInfo{ + crdDirs: []string{dirKcp}, + }, + ClusterTypeSkr: &clusterInfo{ + crdDirs: []string{dirSkr}, + }, + ClusterTypeGarden: &clusterInfo{ + crdDirs: []string{dirGarden}, + }, + }, + } + + for name, cluster := range infra.clusters { + ginkgo.By(fmt.Sprintf("Startig cluster %s", name)) + sch, ok := schemeMap[name] + if !ok { + return nil, fmt.Errorf("missing scheme for cluster %s", name) + } + + env, cfg, err := startCluster(cluster.crdDirs, projectRoot, envtestK8sVersion) + if err != nil { + return nil, fmt.Errorf("error starting cluster %s: %w", name, err) + } + + k8sClient, err := ctrlclient.New(cfg, ctrlclient.Options{Scheme: sch}) + if err != nil { + return nil, fmt.Errorf("error creating client for %s: %w", name, err) + } + + cluster.env = env + cluster.cfg = cfg + cluster.scheme = sch + cluster.client = k8sClient + + ce := &clusterEnv{} + switch name { + case ClusterTypeKcp: + ce.namespace = "kcp-system" + case ClusterTypeSkr: + ce.namespace = "kyma-system" + case ClusterTypeGarden: + ce.namespace = "garden-kyma" + } + cluster.ClusterEnv = ce + } + + ginkgo.By("All started") + + // Create ENV + kcpMgr, err := ctrl.NewManager(infra.clusters[ClusterTypeKcp].cfg, ctrl.Options{ + Scheme: infra.KCP().Scheme(), + Client: ctrlclient.Options{ + Cache: &ctrlclient.CacheOptions{ + Unstructured: true, + }, + }, + }) + if err != nil { + return nil, fmt.Errorf("error creating KCP manager: %w", err) + } + + registry := skrruntime.NewRegistry(infra.SKR().Scheme()) + looper := skrruntime.NewLooper(kcpMgr, infra.SKR().Scheme(), registry, kcpMgr.GetLogger()) + + awsMock := awsmock.New() + + infra.InfraEnv = &infraEnv{ + kcpManager: kcpMgr, + registry: registry, + looper: looper, + awsMock: awsMock, + } + + // Create DSL + infra.InfraDSL = &infraDSL{i: infra} + for _, c := range infra.clusters { + c.ClusterDSL = &clusterDSL{ + ci: c, + ctx: infra.Ctx, + } + } + + return infra, nil +} + +func startCluster(crdsDirs []string, projectRoot, envtestK8sVersion string) (*envtest.Environment, *rest.Config, error) { + env := &envtest.Environment{ + CRDDirectoryPaths: crdsDirs, + ErrorIfCRDPathMissing: true, + BinaryAssetsDirectory: filepath.Join(projectRoot, "bin", "k8s", + fmt.Sprintf("%s-%s-%s", envtestK8sVersion, goruntime.GOOS, goruntime.GOARCH)), + } + + cfg, err := env.Start() + + return env, cfg, err +} diff --git a/components/kcp/pkg/testinfra/scheme.go b/components/kcp/pkg/testinfra/scheme.go new file mode 100644 index 000000000..47a9f36c3 --- /dev/null +++ b/components/kcp/pkg/testinfra/scheme.go @@ -0,0 +1,29 @@ +package testinfra + +import ( + gardenapi "github.com/gardener/gardener/pkg/apis/core/v1beta1" + cloudcontrolv1beta1 "github.com/kyma-project/cloud-manager/components/kcp/api/cloud-control/v1beta1" + cloudresourcesv1beta1 "github.com/kyma-project/cloud-manager/components/kcp/api/cloud-resources/v1beta1" + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" +) + +var schemeMap map[ClusterType]*runtime.Scheme + +func init() { + schemeMap = map[ClusterType]*runtime.Scheme{ + ClusterTypeKcp: runtime.NewScheme(), + ClusterTypeSkr: runtime.NewScheme(), + ClusterTypeGarden: runtime.NewScheme(), + } + // KCP + utilruntime.Must(clientgoscheme.AddToScheme(schemeMap[ClusterTypeKcp])) + utilruntime.Must(cloudcontrolv1beta1.AddToScheme(schemeMap[ClusterTypeKcp])) + // SKR + utilruntime.Must(clientgoscheme.AddToScheme(schemeMap[ClusterTypeSkr])) + utilruntime.Must(cloudresourcesv1beta1.AddToScheme(schemeMap[ClusterTypeSkr])) + // Garden + utilruntime.Must(clientgoscheme.AddToScheme(schemeMap[ClusterTypeGarden])) + utilruntime.Must(gardenapi.AddToScheme(schemeMap[ClusterTypeGarden])) +} diff --git a/components/kcp/pkg/testinfra/suite.go b/components/kcp/pkg/testinfra/suite.go deleted file mode 100644 index 6de149cf5..000000000 --- a/components/kcp/pkg/testinfra/suite.go +++ /dev/null @@ -1,312 +0,0 @@ -package testinfra - -import ( - "context" - "errors" - "fmt" - gardenapi "github.com/gardener/gardener/pkg/apis/core/v1beta1" - "github.com/go-logr/logr" - cloudcontrolv1beta1 "github.com/kyma-project/cloud-manager/components/kcp/api/cloud-control/v1beta1" - cloudresourcesv1beta1 "github.com/kyma-project/cloud-manager/components/kcp/api/cloud-resources/v1beta1" - cloudcontrolcontroller "github.com/kyma-project/cloud-manager/components/kcp/internal/controller/cloud-control" - cloudresourcescontroller "github.com/kyma-project/cloud-manager/components/kcp/internal/controller/cloud-resources" - awsmock "github.com/kyma-project/cloud-manager/components/kcp/pkg/provider/aws/mock" - gcpiprangeclient "github.com/kyma-project/cloud-manager/components/kcp/pkg/provider/gcp/iprange/client" - gcpFilestoreClient "github.com/kyma-project/cloud-manager/components/kcp/pkg/provider/gcp/nfsinstance/client" - skrruntime "github.com/kyma-project/cloud-manager/components/kcp/pkg/skr/runtime" - skrmanager "github.com/kyma-project/cloud-manager/components/kcp/pkg/skr/runtime/manager" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/suite" - "k8s.io/apimachinery/pkg/runtime" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" - clientgoscheme "k8s.io/client-go/kubernetes/scheme" - "k8s.io/client-go/rest" - "k8s.io/klog/v2" - "net/http" - "path/filepath" - goruntime "runtime" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/envtest" - "sigs.k8s.io/controller-runtime/pkg/log/zap" - "sigs.k8s.io/controller-runtime/pkg/manager" - "sync" - "testing" -) - -type ClusterType string - -const ( - ClusterTypeKcp = ClusterType("kcp") - ClusterTypeSkr = ClusterType("skr") - ClusterTypeGarden = ClusterType("garden") -) - -var onceLogger = sync.Once{} - -var schemeMap map[ClusterType]*runtime.Scheme - -var logger logr.Logger - -func init() { - schemeMap = map[ClusterType]*runtime.Scheme{ - ClusterTypeKcp: runtime.NewScheme(), - ClusterTypeSkr: runtime.NewScheme(), - ClusterTypeGarden: runtime.NewScheme(), - } - // KCP - utilruntime.Must(clientgoscheme.AddToScheme(schemeMap[ClusterTypeKcp])) - utilruntime.Must(cloudcontrolv1beta1.AddToScheme(schemeMap[ClusterTypeKcp])) - // SKR - utilruntime.Must(clientgoscheme.AddToScheme(schemeMap[ClusterTypeSkr])) - utilruntime.Must(cloudresourcesv1beta1.AddToScheme(schemeMap[ClusterTypeSkr])) - // Garden - utilruntime.Must(clientgoscheme.AddToScheme(schemeMap[ClusterTypeGarden])) - utilruntime.Must(gardenapi.AddToScheme(schemeMap[ClusterTypeGarden])) -} - -type Cluster struct { - t *testing.T - Name string - Type ClusterType - KymaRef klog.ObjectRef - Cfg *rest.Config - K8sClient client.Client - TestEnv *envtest.Environment - Manager manager.Manager - Cancel context.CancelFunc - Ctx context.Context - Registry skrruntime.SkrRegistry - Runner skrruntime.SkrRunner - awsMock awsmock.Server -} - -type BaseSuite struct { - suite.Suite - kcpClusterName string - clusters map[string]*Cluster - - AwsMock awsmock.Server -} - -func (s *BaseSuite) CommonSetupSuite() { - s.AwsMock = awsmock.New() - onceLogger.Do(func() { - logger = zap.New(zap.UseDevMode(true)) - ctrl.SetLogger(logger) - }) -} - -func (s *BaseSuite) CommonTearDownSuite() { - for name, cluster := range s.clusters { - if cluster.Cancel != nil { - cluster.Cancel() - } - err := cluster.TestEnv.Stop() - if err != nil { - s.T().Errorf("Error stopping testenv %s: %s", name, err) - } - } -} - -func (s *BaseSuite) Cluster(name string) *Cluster { - return s.clusters[name] -} - -type setupClusterOptions struct { - kymaRef *klog.ObjectRef -} - -type SetupClusterOptionFunc = func(options setupClusterOptions) - -func WithKymaRef(namespace, name string) SetupClusterOptionFunc { - return func(options setupClusterOptions) { - options.kymaRef = &klog.ObjectRef{ - Name: name, - Namespace: namespace, - } - } -} - -func (s *BaseSuite) SetupCluster(name string, clusterType ClusterType, opts ...SetupClusterOptionFunc) (*Cluster, error) { - if err := initCrds(); err != nil { - return nil, err - } - - options := setupClusterOptions{} - for _, o := range opts { - o(options) - } - - if clusterType == ClusterTypeKcp { - if len(s.kcpClusterName) > 0 { - return nil, fmt.Errorf("only one KCP cluster can be created, and %s is already started", s.kcpClusterName) - } - s.kcpClusterName = name - } - - if clusterType == ClusterTypeSkr { - if options.kymaRef == nil { - return nil, errors.New("cluster of the SKR type must have WithKymaRef option") - } - if len(s.kcpClusterName) == 0 { - return nil, errors.New("an SKR cluster can se started only once a KCP cluster is already started") - } - } - - cluster := &Cluster{ - t: s.T(), - Type: clusterType, - Name: name, - awsMock: s.AwsMock, - } - - // TestEnv - crdMap := map[ClusterType]string{ - ClusterTypeKcp: dirKcp, - ClusterTypeSkr: dirSkr, - ClusterTypeGarden: dirGarden, - } - dir, ok := crdMap[clusterType] - if !ok { - return nil, fmt.Errorf("unknown crd dir for cluster type %s", clusterType) - } - cluster.TestEnv = &envtest.Environment{ - CRDDirectoryPaths: []string{dir}, - ErrorIfCRDPathMissing: true, - BinaryAssetsDirectory: filepath.Join("..", "..", "bin", "k8s", - fmt.Sprintf("1.28.3-%s-%s", goruntime.GOOS, goruntime.GOARCH)), - } - - // Cfg - cfg, err := cluster.TestEnv.Start() - if err != nil { - return nil, err - } - if cfg == nil { - return nil, errors.New("testenv started with nil cfg") - } - cluster.Cfg = cfg - - // Client - sch, ok := schemeMap[clusterType] - if !ok { - return nil, fmt.Errorf("unknown scheme for cluster type %s", clusterType) - } - k8sClient, err := client.New(cfg, client.Options{Scheme: sch}) - if err != nil { - return nil, err - } - if k8sClient == nil { - return nil, errors.New("got nil k8sClient") - } - cluster.K8sClient = k8sClient - - // Ctx - ctx, cancel := context.WithCancel(context.TODO()) - cluster.Ctx = ctx - cluster.Cancel = cancel - - // KymaRef - cluster.KymaRef.Name = options.kymaRef.Name - cluster.KymaRef.Namespace = options.kymaRef.Namespace - - // Manager controller-runtime for KCP - if clusterType == ClusterTypeKcp { - mngr, err := ctrl.NewManager(cfg, ctrl.Options{ - Scheme: sch, - }) - if err != nil { - return nil, fmt.Errorf("error creating controller-runtime manager: %w", err) - } - cluster.Manager = mngr - } - - // SkrManager for SKR - if clusterType == ClusterTypeSkr { - mngr, err := skrmanager.New(cfg, sch, cluster.KymaRef, logger) - if err != nil { - return nil, fmt.Errorf("error creating SKR Manager: %w", err) - } - cluster.Manager = mngr - cluster.Registry = skrruntime.NewRegistry(sch) - kcpCluster := s.clusters[s.kcpClusterName] - cluster.Runner = skrruntime.NewRunner(cluster.Registry, kcpCluster.Manager) - } - - return cluster, nil -} - -func (s *BaseSuite) RunClusters() { - for _, c := range s.clusters { - c.Run() - } -} - -func (c *Cluster) Run() { - if c.Manager == nil { - return - } - go func() { - err := c.Manager.Start(c.Ctx) - assert.NoError(c.t, err) - }() -} - -func (c *Cluster) SetupAllControllers(awsMock awsmock.Server) error { - switch c.Type { - case ClusterTypeSkr: - return c.setupAllSkrControllers() - case ClusterTypeKcp: - return c.setupAllKcpControllers() - } - return fmt.Errorf("unable to setup cluster %s of type %s", c.Name, c.Type) -} - -func (c *Cluster) setupAllKcpControllers() (err error) { - if err = cloudcontrolcontroller.NewScopeReconciler( - c.Manager, - c.awsMock.ScopeGardenProvider(), - ).SetupWithManager(c.Manager); err != nil { - return err - } - if err = cloudcontrolcontroller.NewIpRangeReconciler( - c.Manager, - c.awsMock.IpRangeSkrProvider(), - func(ctx context.Context, httpClient *http.Client) (gcpiprangeclient.ServiceNetworkingClient, error) { - return nil, nil - }, - func(ctx context.Context, httpClient *http.Client) (gcpiprangeclient.ComputeClient, error) { - return nil, nil - }, - ).SetupWithManager(c.Manager); err != nil { - return err - } - if err = cloudcontrolcontroller.NewNfsInstanceReconciler( - c.Manager, - c.awsMock.NfsInstanceSkrProvider(), - func(ctx context.Context, httpClient *http.Client) (gcpFilestoreClient.FilestoreClient, error) { - return nil, nil - }, - ).SetupWithManager(c.Manager); err != nil { - return err - } - if err = cloudcontrolcontroller.NewVpcPeeringReconciler(c.Manager).SetupWithManager(c.Manager); err != nil { - return err - } - return -} - -func (c *Cluster) setupAllSkrControllers() (err error) { - if err = cloudresourcescontroller.SetupCloudResourcesReconciler(c.Registry); err != nil { - return err - } - if err = cloudresourcescontroller.SetupIpRangeReconciler(c.Registry); err != nil { - return err - } - if err = cloudresourcescontroller.SetupAwsNfsVolumeReconciler(c.Registry); err != nil { - return err - } - - return -} diff --git a/components/kcp/pkg/testinfra/types.go b/components/kcp/pkg/testinfra/types.go new file mode 100644 index 000000000..53cf42f1f --- /dev/null +++ b/components/kcp/pkg/testinfra/types.go @@ -0,0 +1,68 @@ +package testinfra + +import ( + "context" + awsmock "github.com/kyma-project/cloud-manager/components/kcp/pkg/provider/aws/mock" + skrruntime "github.com/kyma-project/cloud-manager/components/kcp/pkg/skr/runtime" + "github.com/kyma-project/cloud-manager/components/kcp/pkg/util" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/rest" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/manager" +) + +type ClusterType string + +const ( + ClusterTypeKcp = ClusterType("kcp") + ClusterTypeSkr = ClusterType("skr") + ClusterTypeGarden = ClusterType("garden") +) + +type Infra interface { + InfraEnv + InfraDSL + + KCP() ClusterInfo + SKR() ClusterInfo + Garden() ClusterInfo + + Stop() error +} + +type ClusterInfo interface { + ClusterEnv + ClusterDSL + + Scheme() *runtime.Scheme + Client() ctrlclient.Client + Cfg() *rest.Config +} + +type InfraEnv interface { + KcpManager() manager.Manager + Registry() skrruntime.SkrRegistry + Looper() skrruntime.SkrLooper + AwsMock() awsmock.Server + + StartControllers(ctx context.Context) + Ctx() context.Context + stopControllers() +} + +type ClusterEnv interface { + Namespace() string + ObjKey(name string) types.NamespacedName +} + +type InfraDSL interface { + GivenKymaCRExists(name string) error + GivenGardenShootAwsExists(name string) error + WhenKymaModuleStateUpdates(kymaName string, state util.KymaModuleState) error +} + +type ClusterDSL interface { + GivenSecretExists(name string, data map[string][]byte) error + GivenNamespaceExists(name string) error +} diff --git a/components/kcp/pkg/util/debugged/doc.go b/components/kcp/pkg/util/debugged/doc.go new file mode 100644 index 000000000..770966ab9 --- /dev/null +++ b/components/kcp/pkg/util/debugged/doc.go @@ -0,0 +1,6 @@ +// Build with debug tag to have isdebugged.Debugged == true, otherwise it will be false +// +// go build -tags=debug a.go +// dlv debug --build-flags='-tags=debug' a.go + +package debugged diff --git a/components/kcp/pkg/util/debugged/no.go b/components/kcp/pkg/util/debugged/no.go new file mode 100644 index 000000000..ac15633e2 --- /dev/null +++ b/components/kcp/pkg/util/debugged/no.go @@ -0,0 +1,6 @@ +//go:build !debug +// +build !debug + +package debugged + +const Debugged = false diff --git a/components/kcp/pkg/util/debugged/when.go b/components/kcp/pkg/util/debugged/when.go new file mode 100644 index 000000000..dcd0639af --- /dev/null +++ b/components/kcp/pkg/util/debugged/when.go @@ -0,0 +1,8 @@ +package debugged + +func When[T any](yes, no T) T { + if Debugged { + return yes + } + return no +} diff --git a/components/kcp/pkg/util/debugged/yes.go b/components/kcp/pkg/util/debugged/yes.go new file mode 100644 index 000000000..c5c988572 --- /dev/null +++ b/components/kcp/pkg/util/debugged/yes.go @@ -0,0 +1,6 @@ +//go:build debug +// +build debug + +package debugged + +const Debugged = true diff --git a/components/kcp/pkg/util/kyma.go b/components/kcp/pkg/util/kyma.go index 4e952fadb..6bb142505 100644 --- a/components/kcp/pkg/util/kyma.go +++ b/components/kcp/pkg/util/kyma.go @@ -1,6 +1,7 @@ package util import ( + "fmt" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" ) @@ -41,6 +42,42 @@ func GetKymaModuleState(k *unstructured.Unstructured, moduleName string) KymaMod return KymaModuleStateNotPresent } +func SetKymaModuleState(k *unstructured.Unstructured, moduleName string, state KymaModuleState) error { + modules, exists, err := unstructured.NestedSlice(k.Object, "status", "modules") + if err != nil { + return err + } + if !exists { + modules = []interface{}{} + err = unstructured.SetNestedSlice(k.Object, modules, "status", "modules") + if err != nil { + return nil + } + } + + for idx, m := range modules { + mm, ok := m.(map[string]interface{}) + if !ok { + return fmt.Errorf("kyma CR module #%d is not a map", idx) + } + + name, exists, err := unstructured.NestedString(mm, "name") + if err != nil { + return err + } + if exists && name == moduleName { + return unstructured.SetNestedField(mm, string(state), "state") + } + } + + modules = append(modules, map[string]interface{}{ + "name": moduleName, + "state": string(state), + }) + + return unstructured.SetNestedSlice(k.Object, modules, "status", "modules") +} + // https://github.com/kyma-project/lifecycle-manager/blob/main/api/shared/state.go // KymaModuleState the state of the modul in the Kyma CR diff --git a/components/lib/composed/action.go b/components/lib/composed/action.go index 018c310da..2ac9d4726 100644 --- a/components/lib/composed/action.go +++ b/components/lib/composed/action.go @@ -31,9 +31,9 @@ func ComposeActions(name string, actions ...Action) Action { lastError = currentCtx.Err() break loop default: - logger. - WithValues("targetAction", actionName). - Info("Running action") + //logger. + // WithValues("targetAction", actionName). + // Info("Running action") err, nextCtx := a(currentCtx, state) lastError = err if nextCtx != nil { @@ -51,7 +51,7 @@ func ComposeActions(name string, actions ...Action) Action { ) if lastError == nil { - l.Info("Reconciliation finished") + //l.Info("Reconciliation finished") return nil, nil } else if fce, ok := lastError.(FlowControlError); ok { l.Info(fmt.Sprintf("Reconciliation finished with flow control: %s", fce)) From 5a66f9f0a6f432d6f2a3288bae425aefe264b52d Mon Sep 17 00:00:00 2001 From: Milos Tomic Date: Mon, 29 Jan 2024 11:55:43 +0100 Subject: [PATCH 3/4] fixing composed tests --- components/lib/composed/action_test.go | 43 -------------------------- 1 file changed, 43 deletions(-) diff --git a/components/lib/composed/action_test.go b/components/lib/composed/action_test.go index a0ddce520..2c6b1a48d 100644 --- a/components/lib/composed/action_test.go +++ b/components/lib/composed/action_test.go @@ -9,7 +9,6 @@ import ( "github.com/go-logr/logr" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" - "github.com/wojas/genericr" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/log" ) @@ -126,48 +125,6 @@ loop: assert.Equal(me.T(), "2.canceled", state.log[3]) } -func (me *composedActionSuite) TestLogsAllRunActions() { - myName := "TestLogsAllRunActions" - - var allLogs []genericr.Entry - ctx := log.IntoContext(me.ctx, logr.New(genericr.New(func(e genericr.Entry) { - allLogs = append(allLogs, e) - }))) - - state := newComposedActionTestState() - - a := ComposeActions( - myName, - buildTestAction("1", nil), - buildTestAction("2", nil), - buildTestAction("3", nil), - ) - - _, _ = a(ctx, state) - - assert.Len(me.T(), allLogs, 4) - - actionName := "github.com/kyma-project/cloud-manager/components/lib/composed.(*composedActionSuite).TestLogsAllRunActions.buildTestAction.func" - - assert.Equal(me.T(), "Running action", allLogs[0].Message) - assert.Equal(me.T(), myName, allLogs[0].FieldsMap()["action"]) - assert.Equal(me.T(), actionName+"2", allLogs[0].FieldsMap()["targetAction"]) - - assert.Equal(me.T(), "Running action", allLogs[1].Message) - assert.Equal(me.T(), myName, allLogs[1].FieldsMap()["action"]) - assert.Equal(me.T(), actionName+"3", allLogs[1].FieldsMap()["targetAction"]) - - assert.Equal(me.T(), "Running action", allLogs[2].Message) - assert.Equal(me.T(), myName, allLogs[2].FieldsMap()["action"]) - assert.Equal(me.T(), actionName+"4", allLogs[2].FieldsMap()["targetAction"]) - - assert.Equal(me.T(), "Reconciliation finished", allLogs[3].Message) - assert.Equal(me.T(), myName, allLogs[3].FieldsMap()["action"]) - assert.Equal(me.T(), actionName+"4", allLogs[3].FieldsMap()["lastAction"]) - assert.Equal(me.T(), nil, allLogs[3].FieldsMap()["result"]) - assert.Equal(me.T(), nil, allLogs[3].FieldsMap()["err"]) -} - func TestComposedAction(t *testing.T) { suite.Run(t, new(composedActionSuite)) } From e2de464dd1a2b8d9ce403bc5f0ff33bc60ca5fdc Mon Sep 17 00:00:00 2001 From: Milos Tomic Date: Mon, 29 Jan 2024 12:18:26 +0100 Subject: [PATCH 4/4] tidy --- components/kcp/go.mod | 2 -- components/kcp/go.sum | 2 -- components/lib/go.mod | 1 - components/lib/go.sum | 3 --- 4 files changed, 8 deletions(-) diff --git a/components/kcp/go.mod b/components/kcp/go.mod index 974e691db..d1020c639 100644 --- a/components/kcp/go.mod +++ b/components/kcp/go.mod @@ -17,7 +17,6 @@ require ( github.com/kyma-project/cloud-manager/components/lib v0.0.0 github.com/onsi/ginkgo/v2 v2.13.0 github.com/onsi/gomega v1.29.0 - github.com/stretchr/testify v1.8.4 google.golang.org/api v0.156.0 k8s.io/api v0.29.0 k8s.io/apimachinery v0.29.0 @@ -74,7 +73,6 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_golang v1.16.0 // indirect github.com/prometheus/client_model v0.4.0 // indirect github.com/prometheus/common v0.44.0 // indirect diff --git a/components/kcp/go.sum b/components/kcp/go.sum index fb63be18e..01977e8af 100644 --- a/components/kcp/go.sum +++ b/components/kcp/go.sum @@ -199,8 +199,6 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/wojas/genericr v0.3.1 h1:AXiDZCbrWcXwaQ17O6udC1TzWhXZ3N2qL70DhTHhi5M= -github.com/wojas/genericr v0.3.1/go.mod h1:q+QMxbjOlWFPsLEaQgbm8ZRTmmoPZanr0yQWENYoFLM= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= diff --git a/components/lib/go.mod b/components/lib/go.mod index 9377c0b0d..662cc7160 100644 --- a/components/lib/go.mod +++ b/components/lib/go.mod @@ -5,7 +5,6 @@ go 1.21.4 require ( github.com/go-logr/logr v1.3.0 github.com/stretchr/testify v1.8.4 - github.com/wojas/genericr v0.3.1 k8s.io/apimachinery v0.29.0 k8s.io/client-go v0.29.0 sigs.k8s.io/controller-runtime v0.16.3 diff --git a/components/lib/go.sum b/components/lib/go.sum index 57d75856d..b97f6e0d5 100644 --- a/components/lib/go.sum +++ b/components/lib/go.sum @@ -14,7 +14,6 @@ github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJ github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= -github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo= @@ -106,8 +105,6 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/wojas/genericr v0.3.1 h1:AXiDZCbrWcXwaQ17O6udC1TzWhXZ3N2qL70DhTHhi5M= -github.com/wojas/genericr v0.3.1/go.mod h1:q+QMxbjOlWFPsLEaQgbm8ZRTmmoPZanr0yQWENYoFLM= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A=