Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[controller] Add spec.lvm to a LocalStorageClass resource #12

Merged
merged 2 commits into from
Mar 5, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
94 changes: 62 additions & 32 deletions crds/localstorageclass.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -29,13 +29,21 @@ spec:
properties:
spec:
type: object
x-kubernetes-validations:
- rule: "!has(self.fileSystem)"
message: "The 'fileSystem' field is not supported yet and cannot be used."
- rule: has(self.lvm) == has(oldSelf.lvm) && has(self.fileSystem) == has(oldSelf.fileSystem)
message: "Modification error: Once defined, 'lvm' or 'fileSystem' configuration cannot be replaced or removed. Ensure the existing storage configuration type is maintained."
description: |
Defines a Kubernetes Storage class configuration.
required:
- type
- reclaimPolicy
- volumeBindingMode
- lvmVolumeGroups
oneOf:
- required:
- lvm
- required:
- fileSystem
properties:
isDefault:
type: boolean
Expand All @@ -44,16 +52,6 @@ spec:
Should this Storage class be used as default.

> Note that the default value is false.
type:
type: string
x-kubernetes-validations:
- rule: self == oldSelf
message: Value is immutable.
description: |
The type of the device.
enum:
- Thick
- Thin
reclaimPolicy:
type: string
x-kubernetes-validations:
Expand All @@ -78,31 +76,63 @@ spec:
enum:
- Immediate
- WaitForFirstConsumer
lvmVolumeGroups:
type: array
x-kubernetes-validations:
- rule: self == oldSelf
message: Value is immutable.
lvm:
type: object
description: |
LVMVolumeGroup resources where Persistent Volume will be create on.
items:
type: object
properties:
name:
type: string
description: |
The LVMVolumeGroup resource's name.
thin:
The field provides a LVM configuration.
required:
- type
- lvmVolumeGroups
properties:
type:
type: string
x-kubernetes-validations:
- rule: self == oldSelf
message: Value is immutable.
description: |
The type of the device.
enum:
- Thick
- Thin
lvmVolumeGroups:
type: array
x-kubernetes-validations:
- rule: self == oldSelf
message: Value is immutable.
description: |
LVMVolumeGroup resources where Persistent Volume will be create on.
items:
type: object
description: |
Thin pool in a LVMVolumeGroup resource.
required:
- name
properties:
poolName:
name:
type: string
description: |
The name of the thin pool.
minLength: 1
pattern: ^.*$
The LVMVolumeGroup resource's name.
thin:
type: object
description: |
Thin pool in a LVMVolumeGroup resource.
required:
- poolName
properties:
poolName:
type: string
description: |
The name of the thin pool.
minLength: 1
pattern: ^.*$
fileSystem:
type: object
x-kubernetes-validations:
- rule: self == oldSelf
message: Value is immutable.
required:
- localPath
properties:
localPath:
type: string
status:
type: object
description: |
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,11 +33,15 @@ type LocalStorageClassList struct {
}

type LocalStorageClassSpec struct {
IsDefault bool `json:"isDefault"`
Type string `json:"type"`
ReclaimPolicy string `json:"reclaimPolicy"`
VolumeBindingMode string `json:"volumeBindingMode"`
LVMVolumeGroups []LocalStorageClassLVG `json:"lvmVolumeGroups"`
IsDefault bool `json:"isDefault"`
ReclaimPolicy string `json:"reclaimPolicy"`
VolumeBindingMode string `json:"volumeBindingMode"`
LVM *LocalStorageClassLVM `json:"lvm,omitempty"`
}

type LocalStorageClassLVM struct {
Type string `json:"type"`
LVMVolumeGroups []LocalStorageClassLVG `json:"lvmVolumeGroups"`
}

type LocalStorageClassStatus struct {
Expand Down
2 changes: 0 additions & 2 deletions images/sds-local-volume-controller/cmd/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,9 +49,7 @@ var (
)

func main() {

ctx := context.Background()

cfgParams := config.NewConfig()

log, err := logger.NewLogger(cfgParams.Loglevel)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -51,10 +51,13 @@ const (
Thin = "Thin"
Thick = "Thick"

Lvm = "lvm"

StorageClassKind = "StorageClass"
StorageClassAPIVersion = "storage.k8s.io/v1"

LocalStorageClassProvisioner = "local.csi.storage.deckhouse.io"
TypeParamKey = LocalStorageClassProvisioner + "/type"
LVMTypeParamKey = LocalStorageClassProvisioner + "/lvm-type"
LVMVolumeBindingModeParamKey = LocalStorageClassProvisioner + "/volume-binding-mode"
LVMVolumeGroupsParamKey = LocalStorageClassProvisioner + "/lvm-volume-groups"
Expand Down Expand Up @@ -149,6 +152,11 @@ func RunLocalStorageClassWatcherController(
shouldRequeue, err := runEventReconcile(ctx, cl, log, scList, lsc)
if err != nil {
log.Error(err, fmt.Sprintf("[CreateFunc] an error occured while reconciles the LocalStorageClass, name: %s", lsc.Name))
err = updateLocalStorageClassPhase(ctx, cl, lsc, FailedStatusPhase, err.Error())
if err != nil {
log.Error(err, fmt.Sprintf("[CreateFunc] unable to update the LocalStorageClass %s", lsc.Name))
shouldRequeue = true
}
}

if shouldRequeue {
Expand Down Expand Up @@ -188,6 +196,11 @@ func RunLocalStorageClassWatcherController(
shouldRequeue, err := runEventReconcile(ctx, cl, log, scList, lsc)
if err != nil {
log.Error(err, fmt.Sprintf("[UpdateFunc] an error occured while reconciles the LocalStorageClass, name: %s", lsc.Name))
err = updateLocalStorageClassPhase(ctx, cl, lsc, FailedStatusPhase, err.Error())
if err != nil {
log.Error(err, fmt.Sprintf("[UpdateFunc] unable to update the LocalStorageClass %s", lsc.Name))
shouldRequeue = true
}
}

if shouldRequeue {
Expand Down Expand Up @@ -431,6 +444,10 @@ func shouldReconcileByUpdateFunc(scList *v1.StorageClassList, lsc *v1alpha1.Loca
}

func shouldReconcileByCreateFunc(scList *v1.StorageClassList, lsc *v1alpha1.LocalStorageClass) bool {
if lsc.DeletionTimestamp != nil {
return false
}

for _, sc := range scList.Items {
if sc.Name == lsc.Name &&
lsc.Status != nil {
Expand Down Expand Up @@ -546,11 +563,23 @@ func configureStorageClass(lsc *v1alpha1.LocalStorageClass) (*v1.StorageClass, e
volumeBindingMode := v1.VolumeBindingMode(lsc.Spec.VolumeBindingMode)
AllowVolumeExpansion := AllowVolumeExpansionDefaultValue

lvgsParam, err := yaml.Marshal(lsc.Spec.LVMVolumeGroups)
if lsc.Spec.LVM == nil {
//TODO: add support for other LSC types
return nil, fmt.Errorf("unable to identify the LocalStorageClass type")
}

lvgsParam, err := yaml.Marshal(lsc.Spec.LVM.LVMVolumeGroups)
if err != nil {
return nil, err
}

params := map[string]string{
TypeParamKey: Lvm,
LVMTypeParamKey: lsc.Spec.LVM.Type,
LVMVolumeBindingModeParamKey: lsc.Spec.VolumeBindingMode,
LVMVolumeGroupsParamKey: string(lvgsParam),
}

isDefault := "false"
if lsc.Spec.IsDefault {
isDefault = "true"
Expand All @@ -568,12 +597,8 @@ func configureStorageClass(lsc *v1alpha1.LocalStorageClass) (*v1.StorageClass, e
DefaultStorageClassAnnotationKey: isDefault,
},
},
Provisioner: LocalStorageClassProvisioner,
Parameters: map[string]string{
LVMTypeParamKey: lsc.Spec.Type,
LVMVolumeBindingModeParamKey: lsc.Spec.VolumeBindingMode,
LVMVolumeGroupsParamKey: string(lvgsParam),
},
Provisioner: LocalStorageClassProvisioner,
Parameters: params,
ReclaimPolicy: &reclaimPolicy,
AllowVolumeExpansion: &AllowVolumeExpansion,
VolumeBindingMode: &volumeBindingMode,
Expand Down Expand Up @@ -637,30 +662,36 @@ func validateLocalStorageClass(
return valid, failedMsgBuilder.String()
}

nonexistentLVGs := findNonexistentLVGs(lvgList, lsc)
if len(nonexistentLVGs) != 0 {
valid = false
failedMsgBuilder.WriteString(fmt.Sprintf("Some of selected LVMVolumeGroups are nonexistent, LVG names: %s\n", strings.Join(nonexistentLVGs, ",")))
}

LVGsFromTheSameNode := findLVMVolumeGroupsOnTheSameNode(lvgList, lsc)
if len(LVGsFromTheSameNode) != 0 {
valid = false
failedMsgBuilder.WriteString(fmt.Sprintf("Some LVMVolumeGroups use the same node, LVG names: %s\n", strings.Join(LVGsFromTheSameNode, "")))
}

if lsc.Spec.Type == Thin {
LVGSWithNonexistentTps := findNonexistentThinPools(lvgList, lsc)
if len(LVGSWithNonexistentTps) != 0 {
if lsc.Spec.LVM != nil {
LVGsFromTheSameNode := findLVMVolumeGroupsOnTheSameNode(lvgList, lsc)
if len(LVGsFromTheSameNode) != 0 {
valid = false
failedMsgBuilder.WriteString(fmt.Sprintf("Some LVMVolumeGroups use nonexistent thin pools, LVG names: %s\n", strings.Join(LVGSWithNonexistentTps, ",")))
failedMsgBuilder.WriteString(fmt.Sprintf("Some LVMVolumeGroups use the same node, LVG names: %s\n", strings.Join(LVGsFromTheSameNode, "")))
}
} else {
LVGsWithTps := findAnyThinPool(lsc)
if len(LVGsWithTps) != 0 {

nonexistentLVGs := findNonexistentLVGs(lvgList, lsc)
if len(nonexistentLVGs) != 0 {
valid = false
failedMsgBuilder.WriteString(fmt.Sprintf("Some LVMVolumeGroups use thin pools though device type is Thick, LVG names: %s\n", strings.Join(LVGsWithTps, ",")))
failedMsgBuilder.WriteString(fmt.Sprintf("Some of selected LVMVolumeGroups are nonexistent, LVG names: %s\n", strings.Join(nonexistentLVGs, ",")))
}

if lsc.Spec.LVM.Type == Thin {
LVGSWithNonexistentTps := findNonexistentThinPools(lvgList, lsc)
if len(LVGSWithNonexistentTps) != 0 {
valid = false
failedMsgBuilder.WriteString(fmt.Sprintf("Some LVMVolumeGroups use nonexistent thin pools, LVG names: %s\n", strings.Join(LVGSWithNonexistentTps, ",")))
}
} else {
LVGsWithTps := findAnyThinPool(lsc)
if len(LVGsWithTps) != 0 {
valid = false
failedMsgBuilder.WriteString(fmt.Sprintf("Some LVMVolumeGroups use thin pools though device type is Thick, LVG names: %s\n", strings.Join(LVGsWithTps, ",")))
}
}
} else {
// TODO: add support for other types
valid = false
failedMsgBuilder.WriteString(fmt.Sprintf("Unable to identify a type of LocalStorageClass %s", lsc.Name))
}

return valid, failedMsgBuilder.String()
Expand All @@ -677,8 +708,8 @@ func findUnmanagedDuplicatedSC(scList *v1.StorageClassList, lsc *v1alpha1.LocalS
}

func findAnyThinPool(lsc *v1alpha1.LocalStorageClass) []string {
badLvgs := make([]string, 0, len(lsc.Spec.LVMVolumeGroups))
for _, lvs := range lsc.Spec.LVMVolumeGroups {
badLvgs := make([]string, 0, len(lsc.Spec.LVM.LVMVolumeGroups))
for _, lvs := range lsc.Spec.LVM.LVMVolumeGroups {
if lvs.Thin != nil {
badLvgs = append(badLvgs, lvs.Name)
}
Expand All @@ -693,8 +724,8 @@ func findNonexistentThinPools(lvgList *v1alpha1.LvmVolumeGroupList, lsc *v1alpha
lvgs[lvg.Name] = lvg
}

badLvgs := make([]string, 0, len(lsc.Spec.LVMVolumeGroups))
for _, lscLvg := range lsc.Spec.LVMVolumeGroups {
badLvgs := make([]string, 0, len(lsc.Spec.LVM.LVMVolumeGroups))
for _, lscLvg := range lsc.Spec.LVM.LVMVolumeGroups {
if lscLvg.Thin == nil {
badLvgs = append(badLvgs, lscLvg.Name)
continue
Expand Down Expand Up @@ -724,8 +755,8 @@ func findNonexistentLVGs(lvgList *v1alpha1.LvmVolumeGroupList, lsc *v1alpha1.Loc
lvgs[lvg.Name] = struct{}{}
}

nonexistent := make([]string, 0, len(lsc.Spec.LVMVolumeGroups))
for _, lvg := range lsc.Spec.LVMVolumeGroups {
nonexistent := make([]string, 0, len(lsc.Spec.LVM.LVMVolumeGroups))
for _, lvg := range lsc.Spec.LVM.LVMVolumeGroups {
if _, exist := lvgs[lvg.Name]; !exist {
nonexistent = append(nonexistent, lvg.Name)
}
Expand All @@ -735,13 +766,13 @@ func findNonexistentLVGs(lvgList *v1alpha1.LvmVolumeGroupList, lsc *v1alpha1.Loc
}

func findLVMVolumeGroupsOnTheSameNode(lvgList *v1alpha1.LvmVolumeGroupList, lsc *v1alpha1.LocalStorageClass) []string {
nodesWithLVGs := make(map[string][]string, len(lsc.Spec.LVMVolumeGroups))
usedLVGs := make(map[string]struct{}, len(lsc.Spec.LVMVolumeGroups))
for _, lvg := range lsc.Spec.LVMVolumeGroups {
nodesWithLVGs := make(map[string][]string, len(lsc.Spec.LVM.LVMVolumeGroups))
usedLVGs := make(map[string]struct{}, len(lsc.Spec.LVM.LVMVolumeGroups))
for _, lvg := range lsc.Spec.LVM.LVMVolumeGroups {
usedLVGs[lvg.Name] = struct{}{}
}

badLVGs := make([]string, 0, len(lsc.Spec.LVMVolumeGroups))
badLVGs := make([]string, 0, len(lsc.Spec.LVM.LVMVolumeGroups))
for _, lvg := range lvgList.Items {
if _, used := usedLVGs[lvg.Name]; used {
for _, node := range lvg.Status.Nodes {
Expand Down
4 changes: 4 additions & 0 deletions images/sds-local-volume-csi/driver/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,10 @@ func (d *Driver) CreateVolume(ctx context.Context, request *csi.CreateVolumeRequ
d.log.Trace(request.String())
d.log.Trace("========== CreateVolume ============")

if request.GetParameters()[internal.TypeKey] != internal.Lvm {
return nil, status.Error(codes.InvalidArgument, "Unsupported Storage Class type")
}

if len(request.Name) == 0 {
return nil, status.Error(codes.InvalidArgument, "Volume Name cannot be empty")
}
Expand Down
2 changes: 2 additions & 0 deletions images/sds-local-volume-csi/internal/const.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ limitations under the License.
package internal

const (
TypeKey = "local.csi.storage.deckhouse.io/type"
Lvm = "lvm"
LvmTypeKey = "local.csi.storage.deckhouse.io/lvm-type"
BindingModeKey = "local.csi.storage.deckhouse.io/volume-binding-mode"
LvmVolumeGroupKey = "local.csi.storage.deckhouse.io/lvm-volume-groups"
Expand Down