From 39ebe64f69f05b0df42b521efe7e769ed7143858 Mon Sep 17 00:00:00 2001 From: Viktor Kramarenko Date: Mon, 4 Mar 2024 14:35:28 +0300 Subject: [PATCH 1/2] added spec.lvm field for the local_storage_class_watcher.go Signed-off-by: Viktor Kramarenko --- crds/localstorageclass.yaml | 69 ++++++------ .../api/v1alpha1/local_storage_class.go | 14 ++- .../sds-local-volume-controller/cmd/main.go | 2 - .../controller/local_storage_class_watcher.go | 101 +++++++++++------- .../sds-local-volume-csi/driver/controller.go | 4 + images/sds-local-volume-csi/internal/const.go | 2 + 6 files changed, 116 insertions(+), 76 deletions(-) diff --git a/crds/localstorageclass.yaml b/crds/localstorageclass.yaml index 85f17c9d..ee4d0373 100644 --- a/crds/localstorageclass.yaml +++ b/crds/localstorageclass.yaml @@ -32,10 +32,8 @@ spec: description: | Defines a Kubernetes Storage class configuration. required: - - type - reclaimPolicy - volumeBindingMode - - lvmVolumeGroups properties: isDefault: type: boolean @@ -44,16 +42,6 @@ spec: Should this Storage class be used as default. > Note that the default value is false. - type: - type: string - x-kubernetes-validations: - - rule: self == oldSelf - message: Value is immutable. - description: | - The type of the device. - enum: - - Thick - - Thin reclaimPolicy: type: string x-kubernetes-validations: @@ -78,31 +66,48 @@ spec: enum: - Immediate - WaitForFirstConsumer - lvmVolumeGroups: - type: array - x-kubernetes-validations: - - rule: self == oldSelf - message: Value is immutable. + lvm: + type: object description: | - LVMVolumeGroup resources where Persistent Volume will be create on. - items: - type: object - properties: - name: - type: string - description: | - The LVMVolumeGroup resource's name. - thin: + The field provides a LVM configuration. + required: + - type + properties: + type: + type: string + x-kubernetes-validations: + - rule: self == oldSelf + message: Value is immutable. + description: | + The type of the device. + enum: + - Thick + - Thin + lvmVolumeGroups: + type: array + x-kubernetes-validations: + - rule: self == oldSelf + message: Value is immutable. + description: | + LVMVolumeGroup resources where Persistent Volume will be create on. + items: type: object - description: | - Thin pool in a LVMVolumeGroup resource. properties: - poolName: + name: type: string description: | - The name of the thin pool. - minLength: 1 - pattern: ^.*$ + The LVMVolumeGroup resource's name. + thin: + type: object + description: | + Thin pool in a LVMVolumeGroup resource. + properties: + poolName: + type: string + description: | + The name of the thin pool. + minLength: 1 + pattern: ^.*$ status: type: object description: | diff --git a/images/sds-local-volume-controller/api/v1alpha1/local_storage_class.go b/images/sds-local-volume-controller/api/v1alpha1/local_storage_class.go index 8ddc29f8..18bbee34 100644 --- a/images/sds-local-volume-controller/api/v1alpha1/local_storage_class.go +++ b/images/sds-local-volume-controller/api/v1alpha1/local_storage_class.go @@ -33,11 +33,15 @@ type LocalStorageClassList struct { } type LocalStorageClassSpec struct { - IsDefault bool `json:"isDefault"` - Type string `json:"type"` - ReclaimPolicy string `json:"reclaimPolicy"` - VolumeBindingMode string `json:"volumeBindingMode"` - LVMVolumeGroups []LocalStorageClassLVG `json:"lvmVolumeGroups"` + IsDefault bool `json:"isDefault"` + ReclaimPolicy string `json:"reclaimPolicy"` + VolumeBindingMode string `json:"volumeBindingMode"` + LVM *LocalStorageClassLVM `json:"lvm,omitempty"` +} + +type LocalStorageClassLVM struct { + Type string `json:"type"` + LVMVolumeGroups []LocalStorageClassLVG `json:"lvmVolumeGroups"` } type LocalStorageClassStatus struct { diff --git a/images/sds-local-volume-controller/cmd/main.go b/images/sds-local-volume-controller/cmd/main.go index 1aa6ddf2..e92850c8 100644 --- a/images/sds-local-volume-controller/cmd/main.go +++ b/images/sds-local-volume-controller/cmd/main.go @@ -49,9 +49,7 @@ var ( ) func main() { - ctx := context.Background() - cfgParams := config.NewConfig() log, err := logger.NewLogger(cfgParams.Loglevel) diff --git a/images/sds-local-volume-controller/pkg/controller/local_storage_class_watcher.go b/images/sds-local-volume-controller/pkg/controller/local_storage_class_watcher.go index a1acbf51..0fe85eb6 100644 --- a/images/sds-local-volume-controller/pkg/controller/local_storage_class_watcher.go +++ b/images/sds-local-volume-controller/pkg/controller/local_storage_class_watcher.go @@ -51,10 +51,13 @@ const ( Thin = "Thin" Thick = "Thick" + Lvm = "lvm" + StorageClassKind = "StorageClass" StorageClassAPIVersion = "storage.k8s.io/v1" LocalStorageClassProvisioner = "local.csi.storage.deckhouse.io" + TypeParamKey = LocalStorageClassProvisioner + "/type" LVMTypeParamKey = LocalStorageClassProvisioner + "/lvm-type" LVMVolumeBindingModeParamKey = LocalStorageClassProvisioner + "/volume-binding-mode" LVMVolumeGroupsParamKey = LocalStorageClassProvisioner + "/lvm-volume-groups" @@ -149,6 +152,11 @@ func RunLocalStorageClassWatcherController( shouldRequeue, err := runEventReconcile(ctx, cl, log, scList, lsc) if err != nil { log.Error(err, fmt.Sprintf("[CreateFunc] an error occured while reconciles the LocalStorageClass, name: %s", lsc.Name)) + err = updateLocalStorageClassPhase(ctx, cl, lsc, FailedStatusPhase, err.Error()) + if err != nil { + log.Error(err, fmt.Sprintf("[CreateFunc] unable to update the LocalStorageClass %s", lsc.Name)) + shouldRequeue = true + } } if shouldRequeue { @@ -188,6 +196,11 @@ func RunLocalStorageClassWatcherController( shouldRequeue, err := runEventReconcile(ctx, cl, log, scList, lsc) if err != nil { log.Error(err, fmt.Sprintf("[UpdateFunc] an error occured while reconciles the LocalStorageClass, name: %s", lsc.Name)) + err = updateLocalStorageClassPhase(ctx, cl, lsc, FailedStatusPhase, err.Error()) + if err != nil { + log.Error(err, fmt.Sprintf("[UpdateFunc] unable to update the LocalStorageClass %s", lsc.Name)) + shouldRequeue = true + } } if shouldRequeue { @@ -546,11 +559,23 @@ func configureStorageClass(lsc *v1alpha1.LocalStorageClass) (*v1.StorageClass, e volumeBindingMode := v1.VolumeBindingMode(lsc.Spec.VolumeBindingMode) AllowVolumeExpansion := AllowVolumeExpansionDefaultValue - lvgsParam, err := yaml.Marshal(lsc.Spec.LVMVolumeGroups) + if lsc.Spec.LVM == nil { + //TODO: add support for other LSC types + return nil, fmt.Errorf("unable to identify the LocalStorageClass type") + } + + lvgsParam, err := yaml.Marshal(lsc.Spec.LVM.LVMVolumeGroups) if err != nil { return nil, err } + params := map[string]string{ + TypeParamKey: Lvm, + LVMTypeParamKey: lsc.Spec.LVM.Type, + LVMVolumeBindingModeParamKey: lsc.Spec.VolumeBindingMode, + LVMVolumeGroupsParamKey: string(lvgsParam), + } + isDefault := "false" if lsc.Spec.IsDefault { isDefault = "true" @@ -568,12 +593,8 @@ func configureStorageClass(lsc *v1alpha1.LocalStorageClass) (*v1.StorageClass, e DefaultStorageClassAnnotationKey: isDefault, }, }, - Provisioner: LocalStorageClassProvisioner, - Parameters: map[string]string{ - LVMTypeParamKey: lsc.Spec.Type, - LVMVolumeBindingModeParamKey: lsc.Spec.VolumeBindingMode, - LVMVolumeGroupsParamKey: string(lvgsParam), - }, + Provisioner: LocalStorageClassProvisioner, + Parameters: params, ReclaimPolicy: &reclaimPolicy, AllowVolumeExpansion: &AllowVolumeExpansion, VolumeBindingMode: &volumeBindingMode, @@ -637,30 +658,36 @@ func validateLocalStorageClass( return valid, failedMsgBuilder.String() } - nonexistentLVGs := findNonexistentLVGs(lvgList, lsc) - if len(nonexistentLVGs) != 0 { - valid = false - failedMsgBuilder.WriteString(fmt.Sprintf("Some of selected LVMVolumeGroups are nonexistent, LVG names: %s\n", strings.Join(nonexistentLVGs, ","))) - } - - LVGsFromTheSameNode := findLVMVolumeGroupsOnTheSameNode(lvgList, lsc) - if len(LVGsFromTheSameNode) != 0 { - valid = false - failedMsgBuilder.WriteString(fmt.Sprintf("Some LVMVolumeGroups use the same node, LVG names: %s\n", strings.Join(LVGsFromTheSameNode, ""))) - } - - if lsc.Spec.Type == Thin { - LVGSWithNonexistentTps := findNonexistentThinPools(lvgList, lsc) - if len(LVGSWithNonexistentTps) != 0 { + if lsc.Spec.LVM != nil { + LVGsFromTheSameNode := findLVMVolumeGroupsOnTheSameNode(lvgList, lsc) + if len(LVGsFromTheSameNode) != 0 { valid = false - failedMsgBuilder.WriteString(fmt.Sprintf("Some LVMVolumeGroups use nonexistent thin pools, LVG names: %s\n", strings.Join(LVGSWithNonexistentTps, ","))) + failedMsgBuilder.WriteString(fmt.Sprintf("Some LVMVolumeGroups use the same node, LVG names: %s\n", strings.Join(LVGsFromTheSameNode, ""))) } - } else { - LVGsWithTps := findAnyThinPool(lsc) - if len(LVGsWithTps) != 0 { + + nonexistentLVGs := findNonexistentLVGs(lvgList, lsc) + if len(nonexistentLVGs) != 0 { valid = false - failedMsgBuilder.WriteString(fmt.Sprintf("Some LVMVolumeGroups use thin pools though device type is Thick, LVG names: %s\n", strings.Join(LVGsWithTps, ","))) + failedMsgBuilder.WriteString(fmt.Sprintf("Some of selected LVMVolumeGroups are nonexistent, LVG names: %s\n", strings.Join(nonexistentLVGs, ","))) } + + if lsc.Spec.LVM.Type == Thin { + LVGSWithNonexistentTps := findNonexistentThinPools(lvgList, lsc) + if len(LVGSWithNonexistentTps) != 0 { + valid = false + failedMsgBuilder.WriteString(fmt.Sprintf("Some LVMVolumeGroups use nonexistent thin pools, LVG names: %s\n", strings.Join(LVGSWithNonexistentTps, ","))) + } + } else { + LVGsWithTps := findAnyThinPool(lsc) + if len(LVGsWithTps) != 0 { + valid = false + failedMsgBuilder.WriteString(fmt.Sprintf("Some LVMVolumeGroups use thin pools though device type is Thick, LVG names: %s\n", strings.Join(LVGsWithTps, ","))) + } + } + } else { + // TODO: add support for other types + valid = false + failedMsgBuilder.WriteString(fmt.Sprintf("Unable to identify a type of LocalStorageClass %s", lsc.Name)) } return valid, failedMsgBuilder.String() @@ -677,8 +704,8 @@ func findUnmanagedDuplicatedSC(scList *v1.StorageClassList, lsc *v1alpha1.LocalS } func findAnyThinPool(lsc *v1alpha1.LocalStorageClass) []string { - badLvgs := make([]string, 0, len(lsc.Spec.LVMVolumeGroups)) - for _, lvs := range lsc.Spec.LVMVolumeGroups { + badLvgs := make([]string, 0, len(lsc.Spec.LVM.LVMVolumeGroups)) + for _, lvs := range lsc.Spec.LVM.LVMVolumeGroups { if lvs.Thin != nil { badLvgs = append(badLvgs, lvs.Name) } @@ -693,8 +720,8 @@ func findNonexistentThinPools(lvgList *v1alpha1.LvmVolumeGroupList, lsc *v1alpha lvgs[lvg.Name] = lvg } - badLvgs := make([]string, 0, len(lsc.Spec.LVMVolumeGroups)) - for _, lscLvg := range lsc.Spec.LVMVolumeGroups { + badLvgs := make([]string, 0, len(lsc.Spec.LVM.LVMVolumeGroups)) + for _, lscLvg := range lsc.Spec.LVM.LVMVolumeGroups { if lscLvg.Thin == nil { badLvgs = append(badLvgs, lscLvg.Name) continue @@ -724,8 +751,8 @@ func findNonexistentLVGs(lvgList *v1alpha1.LvmVolumeGroupList, lsc *v1alpha1.Loc lvgs[lvg.Name] = struct{}{} } - nonexistent := make([]string, 0, len(lsc.Spec.LVMVolumeGroups)) - for _, lvg := range lsc.Spec.LVMVolumeGroups { + nonexistent := make([]string, 0, len(lsc.Spec.LVM.LVMVolumeGroups)) + for _, lvg := range lsc.Spec.LVM.LVMVolumeGroups { if _, exist := lvgs[lvg.Name]; !exist { nonexistent = append(nonexistent, lvg.Name) } @@ -735,13 +762,13 @@ func findNonexistentLVGs(lvgList *v1alpha1.LvmVolumeGroupList, lsc *v1alpha1.Loc } func findLVMVolumeGroupsOnTheSameNode(lvgList *v1alpha1.LvmVolumeGroupList, lsc *v1alpha1.LocalStorageClass) []string { - nodesWithLVGs := make(map[string][]string, len(lsc.Spec.LVMVolumeGroups)) - usedLVGs := make(map[string]struct{}, len(lsc.Spec.LVMVolumeGroups)) - for _, lvg := range lsc.Spec.LVMVolumeGroups { + nodesWithLVGs := make(map[string][]string, len(lsc.Spec.LVM.LVMVolumeGroups)) + usedLVGs := make(map[string]struct{}, len(lsc.Spec.LVM.LVMVolumeGroups)) + for _, lvg := range lsc.Spec.LVM.LVMVolumeGroups { usedLVGs[lvg.Name] = struct{}{} } - badLVGs := make([]string, 0, len(lsc.Spec.LVMVolumeGroups)) + badLVGs := make([]string, 0, len(lsc.Spec.LVM.LVMVolumeGroups)) for _, lvg := range lvgList.Items { if _, used := usedLVGs[lvg.Name]; used { for _, node := range lvg.Status.Nodes { diff --git a/images/sds-local-volume-csi/driver/controller.go b/images/sds-local-volume-csi/driver/controller.go index 02d67056..dbd0b337 100644 --- a/images/sds-local-volume-csi/driver/controller.go +++ b/images/sds-local-volume-csi/driver/controller.go @@ -38,6 +38,10 @@ func (d *Driver) CreateVolume(ctx context.Context, request *csi.CreateVolumeRequ d.log.Trace(request.String()) d.log.Trace("========== CreateVolume ============") + if request.GetParameters()[internal.TypeKey] != internal.Lvm { + return nil, status.Error(codes.InvalidArgument, "Unsupported Storage Class type") + } + if len(request.Name) == 0 { return nil, status.Error(codes.InvalidArgument, "Volume Name cannot be empty") } diff --git a/images/sds-local-volume-csi/internal/const.go b/images/sds-local-volume-csi/internal/const.go index 73248269..fd2506fc 100644 --- a/images/sds-local-volume-csi/internal/const.go +++ b/images/sds-local-volume-csi/internal/const.go @@ -17,6 +17,8 @@ limitations under the License. package internal const ( + TypeKey = "local.csi.storage.deckhouse.io/type" + Lvm = "lvm" LvmTypeKey = "local.csi.storage.deckhouse.io/lvm-type" BindingModeKey = "local.csi.storage.deckhouse.io/volume-binding-mode" LvmVolumeGroupKey = "local.csi.storage.deckhouse.io/lvm-volume-groups" From d0d150282ad08b12deaa9d7ad02737f6029c623c Mon Sep 17 00:00:00 2001 From: Aleksandr Zimin Date: Tue, 5 Mar 2024 00:50:12 +0300 Subject: [PATCH 2/2] Some fixes Signed-off-by: Aleksandr Zimin --- crds/localstorageclass.yaml | 25 +++++++++++++++++++ .../controller/local_storage_class_watcher.go | 4 +++ 2 files changed, 29 insertions(+) diff --git a/crds/localstorageclass.yaml b/crds/localstorageclass.yaml index ee4d0373..5519c137 100644 --- a/crds/localstorageclass.yaml +++ b/crds/localstorageclass.yaml @@ -29,11 +29,21 @@ spec: properties: spec: type: object + x-kubernetes-validations: + - rule: "!has(self.fileSystem)" + message: "The 'fileSystem' field is not supported yet and cannot be used." + - rule: has(self.lvm) == has(oldSelf.lvm) && has(self.fileSystem) == has(oldSelf.fileSystem) + message: "Modification error: Once defined, 'lvm' or 'fileSystem' configuration cannot be replaced or removed. Ensure the existing storage configuration type is maintained." description: | Defines a Kubernetes Storage class configuration. required: - reclaimPolicy - volumeBindingMode + oneOf: + - required: + - lvm + - required: + - fileSystem properties: isDefault: type: boolean @@ -72,6 +82,7 @@ spec: The field provides a LVM configuration. required: - type + - lvmVolumeGroups properties: type: type: string @@ -92,6 +103,8 @@ spec: LVMVolumeGroup resources where Persistent Volume will be create on. items: type: object + required: + - name properties: name: type: string @@ -101,6 +114,8 @@ spec: type: object description: | Thin pool in a LVMVolumeGroup resource. + required: + - poolName properties: poolName: type: string @@ -108,6 +123,16 @@ spec: The name of the thin pool. minLength: 1 pattern: ^.*$ + fileSystem: + type: object + x-kubernetes-validations: + - rule: self == oldSelf + message: Value is immutable. + required: + - localPath + properties: + localPath: + type: string status: type: object description: | diff --git a/images/sds-local-volume-controller/pkg/controller/local_storage_class_watcher.go b/images/sds-local-volume-controller/pkg/controller/local_storage_class_watcher.go index 0fe85eb6..e6a3c8f2 100644 --- a/images/sds-local-volume-controller/pkg/controller/local_storage_class_watcher.go +++ b/images/sds-local-volume-controller/pkg/controller/local_storage_class_watcher.go @@ -444,6 +444,10 @@ func shouldReconcileByUpdateFunc(scList *v1.StorageClassList, lsc *v1alpha1.Loca } func shouldReconcileByCreateFunc(scList *v1.StorageClassList, lsc *v1alpha1.LocalStorageClass) bool { + if lsc.DeletionTimestamp != nil { + return false + } + for _, sc := range scList.Items { if sc.Name == lsc.Name && lsc.Status != nil {