diff --git a/images/sds-lvm-csi/api/v1alpha1/lvm_logical_volume.go b/images/sds-lvm-csi/api/v1alpha1/lvm_logical_volume.go index ed904b2d..6f731809 100644 --- a/images/sds-lvm-csi/api/v1alpha1/lvm_logical_volume.go +++ b/images/sds-lvm-csi/api/v1alpha1/lvm_logical_volume.go @@ -21,22 +21,22 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -type LvmLogicalVolumeList struct { +type LVMLogicalVolumeList struct { metav1.TypeMeta `json:",inline"` metav1.ListMeta `json:"metadata"` - Items []LvmLogicalVolume `json:"items"` + Items []LVMLogicalVolume `json:"items"` } -type LvmLogicalVolume struct { +type LVMLogicalVolume struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` - Spec LvmLogicalVolumeSpec `json:"spec"` - Status *LvmLogicalVolumeStatus `json:"status,omitempty"` + Spec LVMLogicalVolumeSpec `json:"spec"` + Status *LVMLogicalVolumeStatus `json:"status,omitempty"` } -type LvmLogicalVolumeSpec struct { +type LVMLogicalVolumeSpec struct { Type string `json:"type"` Size resource.Quantity `json:"size"` LvmVolumeGroup string `json:"lvmVolumeGroup"` @@ -47,7 +47,7 @@ type ThinLogicalVolumeSpec struct { PoolName string `json:"poolName"` } -type LvmLogicalVolumeStatus struct { +type LVMLogicalVolumeStatus struct { Phase string `json:"phase"` Reason string `json:"reason"` ActualSize resource.Quantity `json:"actualSize"` diff --git a/images/sds-lvm-csi/api/v1alpha1/register.go b/images/sds-lvm-csi/api/v1alpha1/register.go index 441a3be7..7a466122 100644 --- a/images/sds-lvm-csi/api/v1alpha1/register.go +++ b/images/sds-lvm-csi/api/v1alpha1/register.go @@ -24,7 +24,7 @@ import ( const ( LVMVolumeGroupKind = "LvmVolumeGroup" - LVMLogicalVolumeKind = "LvmLogicalVolume" + LVMLogicalVolumeKind = "LVMLogicalVolume" APIGroup = "storage.deckhouse.io" APIVersion = "v1alpha1" TypeMediaAPIVersion = APIGroup + "/" + APIVersion @@ -45,8 +45,8 @@ func addKnownTypes(scheme *runtime.Scheme) error { scheme.AddKnownTypes(SchemeGroupVersion, &LvmVolumeGroup{}, &LvmVolumeGroupList{}, - &LvmLogicalVolume{}, - &LvmLogicalVolumeList{}, + &LVMLogicalVolume{}, + &LVMLogicalVolumeList{}, ) metav1.AddToGroupVersion(scheme, SchemeGroupVersion) return nil diff --git a/images/sds-lvm-csi/api/v1alpha1/zz_generated.deepcopy.go b/images/sds-lvm-csi/api/v1alpha1/zz_generated.deepcopy.go index 1afd3059..3ae15d13 100644 --- a/images/sds-lvm-csi/api/v1alpha1/zz_generated.deepcopy.go +++ b/images/sds-lvm-csi/api/v1alpha1/zz_generated.deepcopy.go @@ -76,10 +76,10 @@ func (in *LvmVolumeGroupList) DeepCopyObject() runtime.Object { return nil } -// -------------- LvmLogicalVolume ---------- +// -------------- LVMLogicalVolume ---------- // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *LvmLogicalVolume) DeepCopyInto(out *LvmLogicalVolume) { +func (in *LVMLogicalVolume) DeepCopyInto(out *LVMLogicalVolume) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) @@ -87,17 +87,17 @@ func (in *LvmLogicalVolume) DeepCopyInto(out *LvmLogicalVolume) { } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EmptyBlockDevice. -func (in *LvmLogicalVolume) DeepCopy() *LvmLogicalVolume { +func (in *LVMLogicalVolume) DeepCopy() *LVMLogicalVolume { if in == nil { return nil } - out := new(LvmLogicalVolume) + out := new(LVMLogicalVolume) in.DeepCopyInto(out) return out } // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *LvmLogicalVolume) DeepCopyObject() runtime.Object { +func (in *LVMLogicalVolume) DeepCopyObject() runtime.Object { if c := in.DeepCopy(); c != nil { return c } @@ -105,13 +105,13 @@ func (in *LvmLogicalVolume) DeepCopyObject() runtime.Object { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *LvmLogicalVolumeList) DeepCopyInto(out *LvmLogicalVolumeList) { +func (in *LVMLogicalVolumeList) DeepCopyInto(out *LVMLogicalVolumeList) { *out = *in out.TypeMeta = in.TypeMeta in.ListMeta.DeepCopyInto(&out.ListMeta) if in.Items != nil { in, out := &in.Items, &out.Items - *out = make([]LvmLogicalVolume, len(*in)) + *out = make([]LVMLogicalVolume, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } @@ -119,17 +119,17 @@ func (in *LvmLogicalVolumeList) DeepCopyInto(out *LvmLogicalVolumeList) { } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GuestbookList. -func (in *LvmLogicalVolumeList) DeepCopy() *LvmLogicalVolumeList { +func (in *LVMLogicalVolumeList) DeepCopy() *LVMLogicalVolumeList { if in == nil { return nil } - out := new(LvmLogicalVolumeList) + out := new(LVMLogicalVolumeList) in.DeepCopyInto(out) return out } // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *LvmLogicalVolumeList) DeepCopyObject() runtime.Object { +func (in *LVMLogicalVolumeList) DeepCopyObject() runtime.Object { if c := in.DeepCopy(); c != nil { return c } diff --git a/images/sds-lvm-csi/driver/controller.go b/images/sds-lvm-csi/driver/controller.go index 0615e697..ce34af11 100644 --- a/images/sds-lvm-csi/driver/controller.go +++ b/images/sds-lvm-csi/driver/controller.go @@ -20,7 +20,7 @@ import ( "context" "errors" "fmt" - "sds-lvm-csi/api/v1alpha1" + "sds-lvm-csi/internal" "sds-lvm-csi/pkg/utils" kerrors "k8s.io/apimachinery/pkg/api/errors" @@ -28,7 +28,6 @@ import ( "github.com/container-storage-interface/spec/lib/go/csi" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - "gopkg.in/yaml.v3" "k8s.io/apimachinery/pkg/api/resource" ) @@ -46,31 +45,24 @@ func (d *Driver) CreateVolume(ctx context.Context, request *csi.CreateVolumeRequ return nil, status.Error(codes.InvalidArgument, "Volume Capability cannot de empty") } - LvmBindingMode := request.GetParameters()[lvmBindingMode] + LvmBindingMode := request.GetParameters()[internal.LvmBindingModeKey] d.log.Info(fmt.Sprintf("storage class LvmBindingMode: %s", LvmBindingMode)) - LvmType := request.GetParameters()[lvmType] + LvmType := request.GetParameters()[internal.LvmTypeKey] d.log.Info(fmt.Sprintf("storage class LvmType: %s", LvmType)) - if len(request.GetParameters()[lvmVolumeGroup]) == 0 { + if len(request.GetParameters()[internal.LvmVolumeGroupKey]) == 0 { err := errors.New("no LVMVolumeGroups specified in a storage class's parameters") d.log.Error(err, fmt.Sprintf("no LVMVolumeGroups were found for the request: %+v", request)) return nil, status.Errorf(codes.InvalidArgument, err.Error()) } - var lvmVolumeGroups LVMVolumeGroups - err := yaml.Unmarshal([]byte(request.GetParameters()[lvmVolumeGroup]), &lvmVolumeGroups) + storageClassLVGs, storageClassLVGParametersMap, err := utils.GetStorageClassLVGsAndParameters(ctx, d.cl, *d.log, request.GetParameters()[internal.LvmVolumeGroupKey]) if err != nil { - d.log.Error(err, "unmarshal yaml lvmVolumeGroup") - } - - lvmVG := make(map[string]string, len(lvmVolumeGroups)) - for _, v := range lvmVolumeGroups { - lvmVG[v.Name] = v.Thin.PoolName + d.log.Error(err, "error GetStorageClassLVGs") + return nil, status.Errorf(codes.Internal, err.Error()) } - d.log.Info(fmt.Sprintf("StorageClass LVM volume groups names: %+v", lvmVG)) - llvName := request.Name d.log.Info(fmt.Sprintf("llv name: %s ", llvName)) @@ -79,52 +71,39 @@ func (d *Driver) CreateVolume(ctx context.Context, request *csi.CreateVolumeRequ var preferredNode string switch LvmBindingMode { - case BindingModeI: - d.log.Info(fmt.Sprintf("LvmBindingMode is %s. Start selecting node", BindingModeI)) - selectedNodeName, freeSpace, err := utils.GetNodeMaxFreeVGSize(ctx, d.cl) + case internal.BindingModeI: + d.log.Info(fmt.Sprintf("LvmBindingMode is %s. Start selecting node", internal.BindingModeI)) + selectedNodeName, freeSpace, err := utils.GetNodeWithMaxFreeSpace(*d.log, storageClassLVGs, storageClassLVGParametersMap, LvmType) if err != nil { d.log.Error(err, "error GetNodeMaxVGSize") } + preferredNode = selectedNodeName - if llvSize.Value() > freeSpace.Value() { - return nil, status.Errorf(codes.Internal, "requested size: %s is greater than free space: %s", llvSize.String(), freeSpace.String()) - } d.log.Info(fmt.Sprintf("Selected node: %s, free space %s ", selectedNodeName, freeSpace.String())) - case BindingModeWFFC: - d.log.Info(fmt.Sprintf("LvmBindingMode is %s. Get preferredNode", BindingModeWFFC)) + if LvmType == internal.LLMTypeThick { + if llvSize.Value() > freeSpace.Value() { + return nil, status.Errorf(codes.Internal, "requested size: %s is greater than free space: %s", llvSize.String(), freeSpace.String()) + } + } + case internal.BindingModeWFFC: + d.log.Info(fmt.Sprintf("LvmBindingMode is %s. Get preferredNode", internal.BindingModeWFFC)) if len(request.AccessibilityRequirements.Preferred) != 0 { t := request.AccessibilityRequirements.Preferred[0].Segments - preferredNode = t[topologyKey] + preferredNode = t[internal.TopologyKey] } } - lvmVolumeGroupName, vgName, err := utils.GetLVMVolumeGroupParams(ctx, d.cl, *d.log, lvmVG, preferredNode, LvmType) + selectedLVG, err := utils.SelectLVG(storageClassLVGs, storageClassLVGParametersMap, preferredNode) if err != nil { - d.log.Error(err, "error GetVGName") + d.log.Error(err, "error SelectLVG") return nil, status.Errorf(codes.Internal, err.Error()) } - d.log.Info(fmt.Sprintf("LvmVolumeGroup: %s", lvmVolumeGroupName)) - d.log.Info(fmt.Sprintf("VGName: %s", vgName)) - d.log.Info(fmt.Sprintf("prefered node: %s", preferredNode)) + llvSpec := utils.GetLLVSpec(*d.log, selectedLVG, storageClassLVGParametersMap, preferredNode, LvmType, *llvSize) + d.log.Info(fmt.Sprintf("LVMLogicalVolumeSpec : %+v", llvSpec)) d.log.Info("------------ CreateLVMLogicalVolume ------------") - var llvThin *v1alpha1.ThinLogicalVolumeSpec - if LvmType == LLVTypeThin { - llvThin = &v1alpha1.ThinLogicalVolumeSpec{} - llvThin.PoolName = lvmVG[lvmVolumeGroupName] - } - - spec := v1alpha1.LvmLogicalVolumeSpec{ - Type: LvmType, - Size: *llvSize, - LvmVolumeGroup: lvmVolumeGroupName, - Thin: llvThin, - } - - d.log.Info(fmt.Sprintf("LvmLogicalVolumeSpec : %+v", spec)) - - _, err = utils.CreateLVMLogicalVolume(ctx, d.cl, llvName, spec) + _, err = utils.CreateLVMLogicalVolume(ctx, d.cl, llvName, llvSpec) if err != nil { if kerrors.IsAlreadyExists(err) { d.log.Info(fmt.Sprintf("LVMLogicalVolume %s already exists", llvName)) @@ -136,7 +115,7 @@ func (d *Driver) CreateVolume(ctx context.Context, request *csi.CreateVolumeRequ d.log.Info("------------ CreateLVMLogicalVolume ------------") d.log.Info("start wait CreateLVMLogicalVolume ") - resizeDelta, err := resource.ParseQuantity(ResizeDelta) + resizeDelta, err := resource.ParseQuantity(internal.ResizeDelta) if err != nil { d.log.Error(err, "error ParseQuantity for ResizeDelta") return nil, err @@ -153,8 +132,8 @@ func (d *Driver) CreateVolume(ctx context.Context, request *csi.CreateVolumeRequ volumeCtx[k] = v } - volumeCtx[subPath] = request.Name - volumeCtx[VGNameKey] = vgName + volumeCtx[internal.SubPath] = request.Name + volumeCtx[internal.VGNameKey] = selectedLVG.Spec.ActualVGNameOnTheNode return &csi.CreateVolumeResponse{ Volume: &csi.Volume{ @@ -164,7 +143,7 @@ func (d *Driver) CreateVolume(ctx context.Context, request *csi.CreateVolumeRequ ContentSource: request.VolumeContentSource, AccessibleTopology: []*csi.Topology{ {Segments: map[string]string{ - topologyKey: preferredNode, + internal.TopologyKey: preferredNode, }}, }, }, @@ -278,7 +257,7 @@ func (d *Driver) ControllerExpandVolume(ctx context.Context, request *csi.Contro return nil, status.Errorf(codes.Internal, "error getting LVMLogicalVolume: %s", err.Error()) } - resizeDelta, err := resource.ParseQuantity(ResizeDelta) + resizeDelta, err := resource.ParseQuantity(internal.ResizeDelta) if err != nil { d.log.Error(err, "error ParseQuantity for ResizeDelta") return nil, err @@ -306,13 +285,15 @@ func (d *Driver) ControllerExpandVolume(ctx context.Context, request *csi.Contro return nil, status.Errorf(codes.Internal, "error getting LVMVolumeGroup: %v", err) } - lvgFreeSpace, err := utils.GetLVMVolumeGroupFreeSpace(*lvg) - if err != nil { - return nil, status.Errorf(codes.Internal, "error getting LVMVolumeGroupCapacity: %v", err) - } + if llv.Spec.Type == internal.LLMTypeThick { + lvgFreeSpace, err := utils.GetLVMVolumeGroupFreeSpace(*lvg) + if err != nil { + return nil, status.Errorf(codes.Internal, "error getting LVMVolumeGroupCapacity: %v", err) + } - if lvgFreeSpace.Value() < (requestCapacity.Value() - llv.Status.ActualSize.Value()) { - return nil, status.Errorf(codes.Internal, "requested size: %s is greater than the capacity of the LVMVolumeGroup: %s", requestCapacity.String(), lvgFreeSpace.String()) + if lvgFreeSpace.Value() < (requestCapacity.Value() - llv.Status.ActualSize.Value()) { + return nil, status.Errorf(codes.Internal, "requested size: %s is greater than the capacity of the LVMVolumeGroup: %s", requestCapacity.String(), lvgFreeSpace.String()) + } } d.log.Info("start resize LVMLogicalVolume") diff --git a/images/sds-lvm-csi/driver/node.go b/images/sds-lvm-csi/driver/node.go index 34359ba0..acb9b037 100644 --- a/images/sds-lvm-csi/driver/node.go +++ b/images/sds-lvm-csi/driver/node.go @@ -19,6 +19,7 @@ package driver import ( "context" "fmt" + "sds-lvm-csi/internal" "github.com/container-storage-interface/spec/lib/go/csi" "google.golang.org/grpc/codes" @@ -41,7 +42,7 @@ func (d *Driver) NodePublishVolume(ctx context.Context, request *csi.NodePublish d.log.Info(request.String()) d.log.Info("------------- NodePublishVolume --------------") - dev := fmt.Sprintf("/dev/%s/%s", request.GetVolumeContext()[VGNameKey], request.VolumeId) + dev := fmt.Sprintf("/dev/%s/%s", request.GetVolumeContext()[internal.VGNameKey], request.VolumeId) var mountOptions []string if request.GetReadonly() { @@ -145,7 +146,7 @@ func (d *Driver) NodeGetInfo(ctx context.Context, request *csi.NodeGetInfoReques MaxVolumesPerNode: 10, AccessibleTopology: &csi.Topology{ Segments: map[string]string{ - topologyKey: d.hostID, + internal.TopologyKey: d.hostID, }, }, }, nil diff --git a/images/sds-lvm-csi/driver/const.go b/images/sds-lvm-csi/internal/const.go similarity index 52% rename from images/sds-lvm-csi/driver/const.go rename to images/sds-lvm-csi/internal/const.go index 7523c5dc..ea549009 100644 --- a/images/sds-lvm-csi/driver/const.go +++ b/images/sds-lvm-csi/internal/const.go @@ -14,19 +14,19 @@ See the License for the specific language governing permissions and limitations under the License. */ -package driver +package internal const ( - lvmType = "lvm.csi.storage.deckhouse.io/lvm-type" - lvmBindingMode = "lvm.csi.storage.deckhouse.io/volume-binding-mode" - lvmVolumeGroup = "lvm.csi.storage.deckhouse.io/lvm-volume-groups" - topologyKey = "topology.sds-lvm-csi/node" - subPath = "subPath" - VGNameKey = "vgname" - LLVTypeThin = "Thin" - LLVTypeThick = "Thick" - LLVStatusCreated = "Created" - BindingModeWFFC = "WaitForFirstConsumer" - BindingModeI = "Immediate" - ResizeDelta = "32Mi" + LvmTypeKey = "lvm.csi.storage.deckhouse.io/lvm-type" + LvmBindingModeKey = "lvm.csi.storage.deckhouse.io/volume-binding-mode" + LvmVolumeGroupKey = "lvm.csi.storage.deckhouse.io/lvm-volume-groups" + TopologyKey = "topology.sds-lvm-csi/node" + SubPath = "subPath" + VGNameKey = "vgname" + LLMTypeThin = "Thin" + LLMTypeThick = "Thick" + LLVStatusCreated = "Created" + BindingModeWFFC = "WaitForFirstConsumer" + BindingModeI = "Immediate" + ResizeDelta = "32Mi" ) diff --git a/images/sds-lvm-csi/pkg/utils/func.go b/images/sds-lvm-csi/pkg/utils/func.go index 74dc10dd..38eae5ed 100644 --- a/images/sds-lvm-csi/pkg/utils/func.go +++ b/images/sds-lvm-csi/pkg/utils/func.go @@ -22,9 +22,11 @@ import ( "fmt" "math" "sds-lvm-csi/api/v1alpha1" + "sds-lvm-csi/internal" "sds-lvm-csi/pkg/logger" "time" + "gopkg.in/yaml.v2" kerrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" @@ -40,9 +42,9 @@ const ( KubernetesApiRequestTimeout = 1 ) -func CreateLVMLogicalVolume(ctx context.Context, kc client.Client, name string, LvmLogicalVolumeSpec v1alpha1.LvmLogicalVolumeSpec) (*v1alpha1.LvmLogicalVolume, error) { +func CreateLVMLogicalVolume(ctx context.Context, kc client.Client, name string, LVMLogicalVolumeSpec v1alpha1.LVMLogicalVolumeSpec) (*v1alpha1.LVMLogicalVolume, error) { var err error - llv := &v1alpha1.LvmLogicalVolume{ + llv := &v1alpha1.LVMLogicalVolume{ TypeMeta: metav1.TypeMeta{ Kind: v1alpha1.LVMLogicalVolumeKind, APIVersion: v1alpha1.TypeMediaAPIVersion, @@ -51,7 +53,7 @@ func CreateLVMLogicalVolume(ctx context.Context, kc client.Client, name string, Name: name, OwnerReferences: []metav1.OwnerReference{}, }, - Spec: LvmLogicalVolumeSpec, + Spec: LVMLogicalVolumeSpec, } for attempt := 0; attempt < KubernetesApiRequestLimit; attempt++ { @@ -68,16 +70,16 @@ func CreateLVMLogicalVolume(ctx context.Context, kc client.Client, name string, } if err != nil { - return nil, fmt.Errorf("after %d attempts of creating LvmLogicalVolume %s, last error: %w", KubernetesApiRequestLimit, name, err) + return nil, fmt.Errorf("after %d attempts of creating LVMLogicalVolume %s, last error: %w", KubernetesApiRequestLimit, name, err) } return llv, nil } -func DeleteLVMLogicalVolume(ctx context.Context, kc client.Client, LvmLogicalVolumeName string) error { +func DeleteLVMLogicalVolume(ctx context.Context, kc client.Client, LVMLogicalVolumeName string) error { var err error - llv := &v1alpha1.LvmLogicalVolume{ + llv := &v1alpha1.LVMLogicalVolume{ ObjectMeta: metav1.ObjectMeta{ - Name: LvmLogicalVolumeName, + Name: LVMLogicalVolumeName, }, TypeMeta: metav1.TypeMeta{ Kind: v1alpha1.LVMLogicalVolumeKind, @@ -94,13 +96,14 @@ func DeleteLVMLogicalVolume(ctx context.Context, kc client.Client, LvmLogicalVol } if err != nil { - return fmt.Errorf("after %d attempts of deleting LvmLogicalVolume %s, last error: %w", KubernetesApiRequestLimit, LvmLogicalVolumeName, err) + return fmt.Errorf("after %d attempts of deleting LVMLogicalVolume %s, last error: %w", KubernetesApiRequestLimit, LVMLogicalVolumeName, err) } return nil } -func WaitForStatusUpdate(ctx context.Context, kc client.Client, log logger.Logger, LvmLogicalVolumeName, namespace string, llvSize, delta resource.Quantity) (int, error) { +func WaitForStatusUpdate(ctx context.Context, kc client.Client, log logger.Logger, LVMLogicalVolumeName, namespace string, llvSize, delta resource.Quantity) (int, error) { var attemptCounter int + sizeEquals := false for { attemptCounter++ select { @@ -109,32 +112,38 @@ func WaitForStatusUpdate(ctx context.Context, kc client.Client, log logger.Logge case <-time.After(500 * time.Millisecond): } - llv, err := GetLVMLogicalVolume(ctx, kc, LvmLogicalVolumeName, namespace) + llv, err := GetLVMLogicalVolume(ctx, kc, LVMLogicalVolumeName, namespace) if err != nil { return attemptCounter, err } - sizeEquals := AreSizesEqualWithinDelta(llvSize, llv.Status.ActualSize, delta) - if attemptCounter%10 == 0 { - log.Info(fmt.Sprintf("[WaitForStatusUpdate] Attempt: %d,LVM Logical Volume status: %+v; delta=%s; sizeEquals=%t", attemptCounter, llv.Status, delta.String(), sizeEquals)) - if llv.Status != nil && llv.Status.Phase == LLVStatusFailed { - return attemptCounter, fmt.Errorf("LVM Logical Volume %s in namespace %s failed", LvmLogicalVolumeName, namespace) - } + log.Info(fmt.Sprintf("[WaitForStatusUpdate] Attempt: %d,LVM Logical Volume: %+v; delta=%s; sizeEquals=%t", attemptCounter, llv, delta.String(), sizeEquals)) } - if llv.Status != nil && llv.Status.Phase == LLVStatusCreated && sizeEquals { - return attemptCounter, nil + + if llv.Status != nil { + sizeEquals = AreSizesEqualWithinDelta(llvSize, llv.Status.ActualSize, delta) + + if llv.Status.Phase == LLVStatusFailed { + return attemptCounter, fmt.Errorf("failed to create LVM logical volume on node for LVMLogicalVolume %s, reason: %s", LVMLogicalVolumeName, llv.Status.Reason) + + } + + if llv.Status.Phase == LLVStatusCreated && sizeEquals { + return attemptCounter, nil + } } + } } -func GetLVMLogicalVolume(ctx context.Context, kc client.Client, LvmLogicalVolumeName, namespace string) (*v1alpha1.LvmLogicalVolume, error) { - var llv v1alpha1.LvmLogicalVolume +func GetLVMLogicalVolume(ctx context.Context, kc client.Client, LVMLogicalVolumeName, namespace string) (*v1alpha1.LVMLogicalVolume, error) { + var llv v1alpha1.LVMLogicalVolume var err error for attempt := 0; attempt < KubernetesApiRequestLimit; attempt++ { err = kc.Get(ctx, client.ObjectKey{ - Name: LvmLogicalVolumeName, + Name: LVMLogicalVolumeName, Namespace: namespace, }, &llv) @@ -143,7 +152,7 @@ func GetLVMLogicalVolume(ctx context.Context, kc client.Client, LvmLogicalVolume } time.Sleep(KubernetesApiRequestTimeout) } - return nil, fmt.Errorf("after %d attempts of getting LvmLogicalVolume %s in namespace %s, last error: %w", KubernetesApiRequestLimit, LvmLogicalVolumeName, namespace, err) + return nil, fmt.Errorf("after %d attempts of getting LVMLogicalVolume %s in namespace %s, last error: %w", KubernetesApiRequestLimit, LVMLogicalVolumeName, namespace, err) } func AreSizesEqualWithinDelta(leftSize, rightSize, allowedDelta resource.Quantity) bool { @@ -153,33 +162,26 @@ func AreSizesEqualWithinDelta(leftSize, rightSize, allowedDelta resource.Quantit return math.Abs(leftSizeFloat-rightSizeFloat) < float64(allowedDelta.Value()) } -func GetNodeMaxFreeVGSize(ctx context.Context, kc client.Client) (nodeName string, freeSpace resource.Quantity, err error) { - listLvgs := &v1alpha1.LvmVolumeGroupList{ - TypeMeta: metav1.TypeMeta{ - Kind: v1alpha1.LVMVolumeGroupKind, - APIVersion: v1alpha1.TypeMediaAPIVersion, - }, - ListMeta: metav1.ListMeta{}, - Items: []v1alpha1.LvmVolumeGroup{}, - } - - for attempt := 0; attempt < KubernetesApiRequestLimit; attempt++ { - err = kc.List(ctx, listLvgs) - if err == nil { - break - } - time.Sleep(KubernetesApiRequestTimeout) - } - - if err != nil { - return "", freeSpace, fmt.Errorf("after %d attempts of getting LvmVolumeGroups, last error: %w", KubernetesApiRequestLimit, err) - } +func GetNodeWithMaxFreeSpace(log logger.Logger, lvgs []v1alpha1.LvmVolumeGroup, storageClassLVGParametersMap map[string]string, lvmType string) (nodeName string, freeSpace resource.Quantity, err error) { var maxFreeSpace int64 - for _, lvg := range listLvgs.Items { - freeSpace, err := GetLVMVolumeGroupFreeSpace(lvg) - if err != nil { - return "", freeSpace, fmt.Errorf("get free space for lvg %s: %w", lvg.Name, err) + for _, lvg := range lvgs { + + switch lvmType { + case internal.LLMTypeThick: + freeSpace, err = GetLVMVolumeGroupFreeSpace(lvg) + if err != nil { + return "", freeSpace, fmt.Errorf("get free space for lvg %+v: %w", lvg, err) + } + case internal.LLMTypeThin: + thinPoolName, ok := storageClassLVGParametersMap[lvg.Name] + if !ok { + return "", freeSpace, fmt.Errorf("thin pool name for lvg %s not found in storage class parameters: %+v", lvg.Name, storageClassLVGParametersMap) + } + freeSpace, err = GetLVMThinPoolFreeSpace(lvg, thinPoolName) + if err != nil { + return "", freeSpace, fmt.Errorf("get free space for thin pool %s in lvg %s: %w", thinPoolName, lvg.Name, err) + } } if freeSpace.Value() > maxFreeSpace { @@ -274,7 +276,31 @@ func GetLVMVolumeGroupFreeSpace(lvg v1alpha1.LvmVolumeGroup) (vgFreeSpace resour return vgFreeSpace, nil } -func UpdateLVMLogicalVolume(ctx context.Context, kc client.Client, llv *v1alpha1.LvmLogicalVolume) error { +func GetLVMThinPoolFreeSpace(lvg v1alpha1.LvmVolumeGroup, thinPoolName string) (thinPoolFreeSpace resource.Quantity, err error) { + var storagePoolThinPool *v1alpha1.StatusThinPool + for _, thinPool := range lvg.Status.ThinPools { + if thinPool.Name == thinPoolName { + storagePoolThinPool = &thinPool + } + } + + if storagePoolThinPool == nil { + return thinPoolFreeSpace, fmt.Errorf("[GetLVMThinPoolFreeSpace] thin pool %s not found in lvg %+v", thinPoolName, lvg) + } + + thinPoolUsedSize, err := resource.ParseQuantity(storagePoolThinPool.UsedSize) + if err != nil { + return thinPoolFreeSpace, fmt.Errorf("[GetLVMThinPoolFreeSpace] parse size thinPool.UsedSize (%s): %w", storagePoolThinPool.UsedSize, err) + } + + thinPoolActualSize := storagePoolThinPool.ActualSize + + thinPoolFreeSpace = thinPoolActualSize.DeepCopy() + thinPoolFreeSpace.Sub(thinPoolUsedSize) + return thinPoolFreeSpace, nil +} + +func UpdateLVMLogicalVolume(ctx context.Context, kc client.Client, llv *v1alpha1.LVMLogicalVolume) error { var err error for attempt := 0; attempt < KubernetesApiRequestLimit; attempt++ { err = kc.Update(ctx, llv) @@ -285,7 +311,88 @@ func UpdateLVMLogicalVolume(ctx context.Context, kc client.Client, llv *v1alpha1 } if err != nil { - return fmt.Errorf("after %d attempts of updating LvmLogicalVolume %s, last error: %w", KubernetesApiRequestLimit, llv.Name, err) + return fmt.Errorf("after %d attempts of updating LVMLogicalVolume %s, last error: %w", KubernetesApiRequestLimit, llv.Name, err) } return nil } + +func GetStorageClassLVGsAndParameters(ctx context.Context, kc client.Client, log logger.Logger, storageClassLVGParametersString string) (storageClassLVGs []v1alpha1.LvmVolumeGroup, storageClassLVGParametersMap map[string]string, err error) { + var storageClassLVGParametersList LVMVolumeGroups + err = yaml.Unmarshal([]byte(storageClassLVGParametersString), &storageClassLVGParametersList) + if err != nil { + log.Error(err, "unmarshal yaml lvmVolumeGroup") + return nil, nil, err + } + + storageClassLVGParametersMap = make(map[string]string, len(storageClassLVGParametersList)) + for _, v := range storageClassLVGParametersList { + storageClassLVGParametersMap[v.Name] = v.Thin.PoolName + } + log.Info(fmt.Sprintf("StorageClass LVM volume groups parameters map: %+v", storageClassLVGParametersMap)) + + lvgs, err := GetLVGList(ctx, kc) + if err != nil { + return nil, nil, err + } + + for _, lvg := range lvgs.Items { + log.Trace(fmt.Sprintf("[GetStorageClassLVGs] process lvg: %+v", lvg)) + + _, ok := storageClassLVGParametersMap[lvg.Name] + if ok { + log.Info(fmt.Sprintf("[GetStorageClassLVGs] found lvg from storage class: %s", lvg.Name)) + log.Info(fmt.Sprintf("[GetStorageClassLVGs] lvg.Status.Nodes[0].Name: %s", lvg.Status.Nodes[0].Name)) + storageClassLVGs = append(storageClassLVGs, lvg) + } else { + log.Trace(fmt.Sprintf("[GetStorageClassLVGs] skip lvg: %s", lvg.Name)) + } + } + + return storageClassLVGs, storageClassLVGParametersMap, nil +} + +func GetLVGList(ctx context.Context, kc client.Client) (*v1alpha1.LvmVolumeGroupList, error) { + var err error + listLvgs := &v1alpha1.LvmVolumeGroupList{ + TypeMeta: metav1.TypeMeta{ + Kind: v1alpha1.LVMVolumeGroupKind, + APIVersion: v1alpha1.TypeMediaAPIVersion, + }, + ListMeta: metav1.ListMeta{}, + Items: []v1alpha1.LvmVolumeGroup{}, + } + + for attempt := 0; attempt < KubernetesApiRequestLimit; attempt++ { + err = kc.List(ctx, listLvgs) + if err == nil { + return listLvgs, nil + } + time.Sleep(KubernetesApiRequestTimeout) + } + + return nil, fmt.Errorf("after %d attempts of getting LvmVolumeGroupList, last error: %w", KubernetesApiRequestLimit, err) +} + +func GetLLVSpec(log logger.Logger, selectedLVG v1alpha1.LvmVolumeGroup, storageClassLVGParametersMap map[string]string, nodeName, lvmType string, llvSize resource.Quantity) v1alpha1.LVMLogicalVolumeSpec { + var llvThin *v1alpha1.ThinLogicalVolumeSpec + if lvmType == internal.LLMTypeThin { + llvThin = &v1alpha1.ThinLogicalVolumeSpec{} + llvThin.PoolName = storageClassLVGParametersMap[selectedLVG.Name] + log.Info(fmt.Sprintf("[GetLLVSpec] Thin pool name: %s", llvThin.PoolName)) + } + return v1alpha1.LVMLogicalVolumeSpec{ + Type: lvmType, + Size: llvSize, + LvmVolumeGroup: selectedLVG.Name, + Thin: llvThin, + } +} + +func SelectLVG(storageClassLVGs []v1alpha1.LvmVolumeGroup, storageClassLVGParametersMap map[string]string, nodeName string) (v1alpha1.LvmVolumeGroup, error) { + for _, lvg := range storageClassLVGs { + if lvg.Status.Nodes[0].Name == nodeName { + return lvg, nil + } + } + return v1alpha1.LvmVolumeGroup{}, fmt.Errorf("[SelectLVG] no LVMVolumeGroup found for node %s", nodeName) +} diff --git a/images/sds-lvm-csi/driver/type.go b/images/sds-lvm-csi/pkg/utils/type.go similarity index 97% rename from images/sds-lvm-csi/driver/type.go rename to images/sds-lvm-csi/pkg/utils/type.go index 4407dbd4..fabf7b56 100644 --- a/images/sds-lvm-csi/driver/type.go +++ b/images/sds-lvm-csi/pkg/utils/type.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package driver +package utils type VolumeGroup struct { Name string `yaml:"name"` diff --git a/templates/webhooks/rbac-for-us.yaml b/templates/webhooks/rbac-for-us.yaml index 04816e9b..7668cde0 100644 --- a/templates/webhooks/rbac-for-us.yaml +++ b/templates/webhooks/rbac-for-us.yaml @@ -12,27 +12,21 @@ metadata: name: d8:{{ .Chart.Name }}:webhooks {{- include "helm_lib_module_labels" (list . (dict "app" "webhooks")) | nindent 2 }} rules: - - verbs: - - get - - list - - watch - apiGroups: - - storage.deckhouse.io - resources: - - drbdstorageclasses - - verbs: - - get - - list - apiGroups: + - apiGroups: - "" - resources: - - nodes - - verbs: + verbs: - get - apiGroups: - - "" resources: + - pods - persistentvolumeclaims + - persistentvolumes + - apiGroups: + - storage.k8s.io + verbs: + - get + resources: + - storageclasses + --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1