From f3d08b1647b6454f1a9c01a7fda857143044f0f6 Mon Sep 17 00:00:00 2001 From: Oksana Grishchenko <91597950+oksana-grishchenko@users.noreply.github.com> Date: Fri, 30 Aug 2024 10:28:16 +0200 Subject: [PATCH] EVEREST-1305 Operator sharding API (#495) --- api/v1alpha1/databasecluster_types.go | 19 +++ api/v1alpha1/zz_generated.deepcopy.go | 36 ++++ .../everest.percona.com_databaseclusters.yaml | 28 ++++ .../everest.percona.com_databaseclusters.yaml | 28 ++++ controllers/providers/psmdb/applier.go | 154 +++++++++++++++--- deploy/bundle.yaml | 27 +++ 6 files changed, 271 insertions(+), 21 deletions(-) diff --git a/api/v1alpha1/databasecluster_types.go b/api/v1alpha1/databasecluster_types.go index 234c11ab2..5945dda14 100644 --- a/api/v1alpha1/databasecluster_types.go +++ b/api/v1alpha1/databasecluster_types.go @@ -314,6 +314,23 @@ type Monitoring struct { Resources corev1.ResourceRequirements `json:"resources,omitempty"` } +type ConfigServer struct { + // Replicas is the amount of configServers + // +kubebuilder:validation:Minimum:=1 + Replicas int32 `json:"replicas"` +} + +// Sharding are the sharding options. Available only for psmdb +type Sharding struct { + // Enabled defines if the sharding is enabled + Enabled bool `json:"enabled"` + // Shards defines the number of shards + // +kubebuilder:validation:Minimum:=1 + Shards int32 `json:"shards"` + // ConfigServer represents the sharding configuration server settings + ConfigServer ConfigServer `json:"configServer"` +} + // DatabaseClusterSpec defines the desired state of DatabaseCluster. type DatabaseClusterSpec struct { // Paused is a flag to stop the cluster @@ -333,6 +350,8 @@ type DatabaseClusterSpec struct { Backup Backup `json:"backup,omitempty"` // Monitoring is the monitoring configuration Monitoring *Monitoring `json:"monitoring,omitempty"` + // Sharding is the sharding configuration. PSMDB-only + Sharding *Sharding `json:"sharding,omitempty"` } // DatabaseClusterStatus defines the observed state of DatabaseCluster. diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 65c1fe64e..bf9123bd1 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -245,6 +245,21 @@ func (in ComponentsMap) DeepCopy() ComponentsMap { return *out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ConfigServer) DeepCopyInto(out *ConfigServer) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConfigServer. +func (in *ConfigServer) DeepCopy() *ConfigServer { + if in == nil { + return nil + } + out := new(ConfigServer) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DataSource) DeepCopyInto(out *DataSource) { *out = *in @@ -541,6 +556,11 @@ func (in *DatabaseClusterSpec) DeepCopyInto(out *DatabaseClusterSpec) { *out = new(Monitoring) (*in).DeepCopyInto(*out) } + if in.Sharding != nil { + in, out := &in.Sharding, &out.Sharding + *out = new(Sharding) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DatabaseClusterSpec. @@ -987,6 +1007,22 @@ func (in *RestoreDate) DeepCopy() *RestoreDate { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Sharding) DeepCopyInto(out *Sharding) { + *out = *in + out.ConfigServer = in.ConfigServer +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Sharding. +func (in *Sharding) DeepCopy() *Sharding { + if in == nil { + return nil + } + out := new(Sharding) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Storage) DeepCopyInto(out *Storage) { *out = *in diff --git a/bundle/manifests/everest.percona.com_databaseclusters.yaml b/bundle/manifests/everest.percona.com_databaseclusters.yaml index 09d02ad70..af3205a26 100644 --- a/bundle/manifests/everest.percona.com_databaseclusters.yaml +++ b/bundle/manifests/everest.percona.com_databaseclusters.yaml @@ -367,6 +367,34 @@ spec: - pgbouncer type: string type: object + sharding: + description: Sharding is the sharding configuration. PSMDB-only + properties: + configServer: + description: ConfigServer represents the sharding configuration + server settings + properties: + replicas: + description: Replicas is the amount of configServers + format: int32 + minimum: 1 + type: integer + required: + - replicas + type: object + enabled: + description: Enabled defines if the sharding is enabled + type: boolean + shards: + description: Shards defines the number of shards + format: int32 + minimum: 1 + type: integer + required: + - configServer + - enabled + - shards + type: object required: - engine type: object diff --git a/config/crd/bases/everest.percona.com_databaseclusters.yaml b/config/crd/bases/everest.percona.com_databaseclusters.yaml index 4121a752b..59542363e 100644 --- a/config/crd/bases/everest.percona.com_databaseclusters.yaml +++ b/config/crd/bases/everest.percona.com_databaseclusters.yaml @@ -367,6 +367,34 @@ spec: - pgbouncer type: string type: object + sharding: + description: Sharding is the sharding configuration. PSMDB-only + properties: + configServer: + description: ConfigServer represents the sharding configuration + server settings + properties: + replicas: + description: Replicas is the amount of configServers + format: int32 + minimum: 1 + type: integer + required: + - replicas + type: object + enabled: + description: Enabled defines if the sharding is enabled + type: boolean + shards: + description: Shards defines the number of shards + format: int32 + minimum: 1 + type: integer + required: + - configServer + - enabled + - shards + type: object required: - engine type: object diff --git a/controllers/providers/psmdb/applier.go b/controllers/providers/psmdb/applier.go index b092b1734..3189f03bb 100644 --- a/controllers/providers/psmdb/applier.go +++ b/controllers/providers/psmdb/applier.go @@ -78,54 +78,166 @@ func (p *applier) Engine() error { Users: database.Spec.Engine.UserSecretsName, EncryptionKey: database.Name + encryptionKeySuffix, } - if database.Spec.Engine.Config != "" { - psmdb.Spec.Replsets[0].Configuration = psmdbv1.MongoConfiguration(database.Spec.Engine.Config) + + p.configureSharding() + + return nil +} + +func (p *applier) configureSharding() { + database := p.DB + psmdb := p.PerconaServerMongoDB + + // TODO: implement disabling + if database.Spec.Sharding == nil || !database.Spec.Sharding.Enabled { + // keep the default configuration + p.configureReplSetSpec(psmdb.Spec.Replsets[0], rsName(0)) + return } - if psmdb.Spec.Replsets[0].Configuration == "" { + + psmdb.Spec.Sharding.Enabled = database.Spec.Sharding.Enabled + + // Add replsets if needed to fit the shards number + // TODO: implement scale down + shards := int(database.Spec.Sharding.Shards) + for i := len(psmdb.Spec.Replsets); i < shards; i++ { + rs0copy := *psmdb.Spec.Replsets[0] + psmdb.Spec.Replsets = append(psmdb.Spec.Replsets, &rs0copy) + } + + // configure all replsets + for i := 0; i < shards; i++ { + p.configureReplSetSpec(psmdb.Spec.Replsets[i], rsName(i)) + } + + if psmdb.Spec.Sharding.ConfigsvrReplSet == nil { + psmdb.Spec.Sharding.ConfigsvrReplSet = &psmdbv1.ReplsetSpec{} + p.configureConfigsvrReplSet(psmdb.Spec.Sharding.ConfigsvrReplSet) + } +} + +func rsName(i int) string { + return fmt.Sprintf("rs%v", i) +} + +func (p *applier) configureConfigsvrReplSet(configsvr *psmdbv1.ReplsetSpec) { + database := p.DB + configsvr.Size = database.Spec.Sharding.ConfigServer.Replicas + configsvr.MultiAZ.Affinity = &psmdbv1.PodAffinity{ + Advanced: common.DefaultAffinitySettings().DeepCopy(), + } + configsvr.VolumeSpec = &psmdbv1.VolumeSpec{ + PersistentVolumeClaim: psmdbv1.PVCSpec{ + PersistentVolumeClaimSpec: &corev1.PersistentVolumeClaimSpec{ + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: p.DB.Spec.Engine.Storage.Size, + }, + }, + }, + }, + } +} + +func (p *applier) configureReplSetSpec(spec *psmdbv1.ReplsetSpec, name string) { + spec.Name = name + dbEngine := p.DB.Spec.Engine + if dbEngine.Config != "" { + spec.Configuration = psmdbv1.MongoConfiguration(dbEngine.Config) + } + if spec.Configuration == "" { // Config missing from the DatabaseCluster CR and the template (if any), apply the default one - psmdb.Spec.Replsets[0].Configuration = psmdbv1.MongoConfiguration(psmdbDefaultConfigurationTemplate) + spec.Configuration = psmdbv1.MongoConfiguration(psmdbDefaultConfigurationTemplate) } affinity := &psmdbv1.PodAffinity{ Advanced: common.DefaultAffinitySettings().DeepCopy(), } - psmdb.Spec.Replsets[0].MultiAZ.Affinity = affinity - - psmdb.Spec.Replsets[0].Size = database.Spec.Engine.Replicas - psmdb.Spec.Replsets[0].VolumeSpec = &psmdbv1.VolumeSpec{ + spec.MultiAZ.Affinity = affinity + spec.Size = dbEngine.Replicas + spec.VolumeSpec = &psmdbv1.VolumeSpec{ PersistentVolumeClaim: psmdbv1.PVCSpec{ PersistentVolumeClaimSpec: &corev1.PersistentVolumeClaimSpec{ - StorageClassName: database.Spec.Engine.Storage.Class, + StorageClassName: dbEngine.Storage.Class, Resources: corev1.VolumeResourceRequirements{ Requests: corev1.ResourceList{ - corev1.ResourceStorage: database.Spec.Engine.Storage.Size, + corev1.ResourceStorage: dbEngine.Storage.Size, }, }, }, }, } - if !database.Spec.Engine.Resources.CPU.IsZero() { - psmdb.Spec.Replsets[0].MultiAZ.Resources.Limits[corev1.ResourceCPU] = database.Spec.Engine.Resources.CPU + if !dbEngine.Resources.CPU.IsZero() { + spec.MultiAZ.Resources.Limits[corev1.ResourceCPU] = dbEngine.Resources.CPU } - if !database.Spec.Engine.Resources.Memory.IsZero() { - psmdb.Spec.Replsets[0].MultiAZ.Resources.Limits[corev1.ResourceMemory] = database.Spec.Engine.Resources.Memory + if !dbEngine.Resources.Memory.IsZero() { + spec.MultiAZ.Resources.Limits[corev1.ResourceMemory] = dbEngine.Resources.Memory } - return nil } func (p *applier) Proxy() error { psmdb := p.PerconaServerMongoDB database := p.DB + + // if sharding is disabled, expose the default replset directly as usual according to db proxy settings + if database.Spec.Sharding == nil || !database.Spec.Sharding.Enabled { + psmdb.Spec.Sharding.Enabled = false + err := p.exposeDefaultReplSet(&psmdb.Spec.Replsets[0].Expose) + if err != nil { + return err + } + } + + // otherwise configure psmdb.Spec.Sharding.Mongos according to the db proxy settings + if psmdb.Spec.Sharding.Mongos == nil { + var size int32 = 1 + if database.Spec.Proxy.Replicas != nil { + size = *database.Spec.Proxy.Replicas + } + psmdb.Spec.Sharding.Mongos = &psmdbv1.MongosSpec{ + Size: size, + } + } + err := p.exposeShardedCluster(&psmdb.Spec.Sharding.Mongos.Expose) + if err != nil { + return err + } + + // disable direct exposure of replsets since .psmdb.Spec.Sharding.Mongos works like proxy + psmdb.Spec.Replsets[0].Expose.Enabled = false + psmdb.Spec.Replsets[0].Expose.ExposeType = corev1.ServiceTypeClusterIP + return nil +} + +func (p *applier) exposeShardedCluster(expose *psmdbv1.MongosExpose) error { + database := p.DB + switch database.Spec.Proxy.Expose.Type { + case everestv1alpha1.ExposeTypeInternal: + expose.ExposeType = corev1.ServiceTypeClusterIP + case everestv1alpha1.ExposeTypeExternal: + expose.ExposeType = corev1.ServiceTypeLoadBalancer + expose.LoadBalancerSourceRanges = database.Spec.Proxy.Expose.IPSourceRangesStringArray() + if annotations, ok := common.ExposeAnnotationsMap[p.clusterType]; ok { + expose.ServiceAnnotations = annotations + } + default: + return fmt.Errorf("invalid expose type %s", database.Spec.Proxy.Expose.Type) + } + return nil +} + +func (p *applier) exposeDefaultReplSet(expose *psmdbv1.ExposeTogglable) error { + database := p.DB switch database.Spec.Proxy.Expose.Type { case everestv1alpha1.ExposeTypeInternal: - psmdb.Spec.Replsets[0].Expose.Enabled = false - psmdb.Spec.Replsets[0].Expose.ExposeType = corev1.ServiceTypeClusterIP + expose.Enabled = false + expose.ExposeType = corev1.ServiceTypeClusterIP case everestv1alpha1.ExposeTypeExternal: - psmdb.Spec.Replsets[0].Expose.Enabled = true - psmdb.Spec.Replsets[0].Expose.ExposeType = corev1.ServiceTypeLoadBalancer - psmdb.Spec.Replsets[0].Expose.LoadBalancerSourceRanges = database.Spec.Proxy.Expose.IPSourceRangesStringArray() + expose.Enabled = true + expose.ExposeType = corev1.ServiceTypeLoadBalancer + expose.LoadBalancerSourceRanges = database.Spec.Proxy.Expose.IPSourceRangesStringArray() if annotations, ok := common.ExposeAnnotationsMap[p.clusterType]; ok { - psmdb.Spec.Replsets[0].Expose.ServiceAnnotations = annotations + expose.ServiceAnnotations = annotations } default: return fmt.Errorf("invalid expose type %s", database.Spec.Proxy.Expose.Type) diff --git a/deploy/bundle.yaml b/deploy/bundle.yaml index 22b69a661..8100ded07 100644 --- a/deploy/bundle.yaml +++ b/deploy/bundle.yaml @@ -711,6 +711,33 @@ spec: - pgbouncer type: string type: object + sharding: + description: Sharding is the sharding configuration. PSMDB-only + properties: + configServer: + description: ConfigServer represents the sharding configuration server settings + properties: + replicas: + description: Replicas is the amount of configServers + format: int32 + minimum: 1 + type: integer + required: + - replicas + type: object + enabled: + description: Enabled defines if the sharding is enabled + type: boolean + shards: + description: Shards defines the number of shards + format: int32 + minimum: 1 + type: integer + required: + - configServer + - enabled + - shards + type: object required: - engine type: object