Skip to content

Commit

Permalink
refactor
Browse files Browse the repository at this point in the history 
Signed-off-by: Aleksandr Zimin <alexandr.zimin@flant.com>
  • Loading branch information
@AleksZimin
AleksZimin committed Apr 24, 2024
1 parent 49501a0 commit 56af844
Show file tree
Hide file tree
Showing 3 changed files with 136 additions and 125 deletions.
27 changes: 12 additions & 15 deletions images/sds-local-volume-csi/driver/node.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,8 +77,6 @@ func (d *Driver) NodeStageVolume(ctx context.Context, request *csi.NodeStageVolu
return &csi.NodeStageVolumeResponse{}, nil
}

isBlock := false

mountVolume := volCap.GetMount()
if mountVolume == nil {
return nil, status.Error(codes.InvalidArgument, "[NodeStageVolume] Volume capability mount cannot be empty")
Expand Down Expand Up @@ -124,7 +122,7 @@ func (d *Driver) NodeStageVolume(ctx context.Context, request *csi.NodeStageVolu
d.log.Trace(fmt.Sprintf("lvmThinPoolName = %s", lvmThinPoolName))
d.log.Trace(fmt.Sprintf("fsType = %s", fsType))

err = d.storeManager.FormatAndMount(devPath, target, isBlock, fsType, false, mountOptions, lvmType, lvmThinPoolName)
err = d.storeManager.NodeStageVolumeFS(devPath, target, fsType, mountOptions, lvmType, lvmThinPoolName)
if err != nil {
d.log.Error(err, "[NodeStageVolume] Error mounting volume")
return nil, status.Errorf(codes.Internal, "[NodeStageVolume] Error format device %q and mounting volume at %q: %v", devPath, target, err)
Expand Down Expand Up @@ -219,8 +217,6 @@ func (d *Driver) NodePublishVolume(ctx context.Context, request *csi.NodePublish
d.inFlight.Delete(volumeID)
}()

fsType := ""

switch volCap.GetAccessType().(type) {
case *csi.VolumeCapability_Block:
d.log.Trace("[NodePublishVolume] Block volume detected.")
Expand All @@ -238,13 +234,13 @@ func (d *Driver) NodePublishVolume(ctx context.Context, request *csi.NodePublish
if !exists {
return nil, status.Errorf(codes.NotFound, "[NodePublishVolume] Device %s not found", devPath)
}
err = d.storeManager.FormatAndMount(devPath, target, true, fsType, false, mountOptions, "", "")
err = d.storeManager.NodePublishVolumeBlock(devPath, target, mountOptions)
if err != nil {
return nil, status.Errorf(codes.Internal, "[NodePublishVolume] Error mounting volume %q at %q: %v", devPath, target, err)
}

case *csi.VolumeCapability_Mount:
d.log.Trace("[NodePublishVolume] Mount volume detected.")
d.log.Trace("[NodePublishVolume] FS type volume detected.")
mountVolume := volCap.GetMount()
if mountVolume == nil {
return nil, status.Error(codes.InvalidArgument, "[NodePublishVolume] Volume capability mount cannot be empty")
Expand All @@ -261,7 +257,7 @@ func (d *Driver) NodePublishVolume(ctx context.Context, request *csi.NodePublish

mountOptions = collectMountOptions(fsType, mountVolume.GetMountFlags(), mountOptions)

err := d.storeManager.BindMount(source, target, fsType, mountOptions)
err := d.storeManager.NodePublishVolumeFS(source, target, fsType, mountOptions)
if err != nil {
return nil, status.Errorf(codes.Internal, "[NodePublishVolume] Error bind mounting volume %q. Source: %q. Target: %q. Mount options:%v. Err: %v", volumeID, source, target, mountOptions, err)
}
Expand Down Expand Up @@ -378,12 +374,13 @@ func collectMountOptions(fsType string, mountFlags, mountOptions []string) []str
}
}

// // By default, xfs does not allow mounting of two volumes with the same filesystem uuid.
// // Force ignore this uuid to be able to mount volume + its clone / restored snapshot on the same node.
// if fsType == FSTypeXfs {
// if !slices.Contains(mountOptions, "nouuid") {
// mountOptions = append(mountOptions, "nouuid")
// }
// }
// By default, xfs does not allow mounting of two volumes with the same filesystem uuid.
// Force ignore this uuid to be able to mount volume + its clone / restored snapshot on the same node.
if fsType == internal.FSTypeXfs {
if !slices.Contains(mountOptions, "nouuid") {
mountOptions = append(mountOptions, "nouuid")
}
}

return mountOptions
}
1 change: 1 addition & 0 deletions images/sds-local-volume-csi/internal/const.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,4 +34,5 @@ const (
ResizeDelta = "32Mi"
// FSTypeExt4 represents the ext4 filesystem type
FSTypeExt4 = "ext4"
FSTypeXfs = "xfs"
)
233 changes: 123 additions & 110 deletions .../sds-local-volume-csi/pkg/utils/volume.go → ...olume-csi/pkg/utils/node-store-manager.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,14 +29,15 @@ import (
)

type NodeStoreManager interface {
FormatAndMount(source, target string, isBlock bool, fsType string, readonly bool, mountOpts []string, lvmType, lvmThinPoolName string) error
NodeStageVolumeFS(source, target string, fsType string, mountOpts []string, lvmType, lvmThinPoolName string) error
NodePublishVolumeBlock(source, target string, mountOpts []string) error
NodePublishVolumeFS(source, target, fsType string, mountOpts []string) error
Unstage(target string) error
Unpublish(target string) error
IsNotMountPoint(target string) (bool, error)
ResizeFS(target string) error
PathExists(path string) (bool, error)
NeedResize(devicePath string, deviceMountPath string) (bool, error)
BindMount(source, target, fsType string, mountOpts []string) error
}

type Store struct {
Expand All @@ -54,109 +55,162 @@ func NewStore(logger *logger.Logger) *Store {
}
}

func (s *Store) FormatAndMount(devSourcePath, target string, isBlock bool, fsType string, readonly bool, mntOpts []string, lvmType, lvmThinPoolName string) error {
s.Log.Info(" ----== Node Mount ==---- ")
func (s *Store) NodeStageVolumeFS(source, target string, fsType string, mountOpts []string, lvmType, lvmThinPoolName string) error {
s.Log.Trace(" ----== Start NodeStageVolumeFS ==---- ")

s.Log.Trace("≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈ Mount options ≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈")
s.Log.Trace(fmt.Sprintf("[mount] params source=%s target=%s fs=%s blockMode=%t mountOptions=%v", devSourcePath, target, fsType, isBlock, mntOpts))
s.Log.Trace(fmt.Sprintf("[mount] params source=%s target=%s fs=%s mountOptions=%v", source, target, fsType, mountOpts))
s.Log.Trace("≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈ Mount options ≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈")

info, err := os.Stat(devSourcePath)
info, err := os.Stat(source)
if err != nil {
return fmt.Errorf("failed to stat source device: %w", err)
}

if (info.Mode() & os.ModeDevice) != os.ModeDevice {
return fmt.Errorf("[NewMount] path %s is not a device", devSourcePath)
return fmt.Errorf("[NewMount] path %s is not a device", source)
}

s.Log.Trace("≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈ MODE SOURCE ≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈")
s.Log.Trace(info.Mode().String())
s.Log.Trace("≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈ MODE SOURCE ≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈")

s.Log.Trace("≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈ isBlock ≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈")
s.Log.Trace(fmt.Sprintf("%t ", isBlock))
s.Log.Trace("≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈ isBlock ≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈")
s.Log.Trace("≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈ FS MOUNT ≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈")
s.Log.Trace("-----------------== start MkdirAll ==-----------------")
s.Log.Trace("mkdir create dir =" + target)
exists, err := s.PathExists(target)
if err != nil {
return fmt.Errorf("[PathExists] could not check if target directory %s exists: %w", target, err)
}
if !exists {
s.Log.Debug(fmt.Sprintf("Creating target directory %s", target))
if err := os.MkdirAll(target, os.FileMode(0755)); err != nil {
return fmt.Errorf("[MkdirAll] could not create target directory %s: %w", target, err)
}
}
s.Log.Trace("-----------------== stop MkdirAll ==-----------------")

if !isBlock {
s.Log.Trace("≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈ FS MOUNT ≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈")
s.Log.Trace("-----------------== start MkdirAll ==-----------------")
s.Log.Trace("mkdir create dir =" + target)
exists, err := s.PathExists(target)
isMountPoint, err := s.NodeStorage.IsMountPoint(target)
if err != nil {
return fmt.Errorf("[s.NodeStorage.IsMountPoint] unable to determine mount status of %s: %w", target, err)
}

s.Log.Trace("≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈ isMountPoint ≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈")
s.Log.Trace(fmt.Sprintf("%t", isMountPoint))
s.Log.Trace("≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈ isMountPoint ≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈")

if isMountPoint {
mapperSourcePath := toMapperPath(source)
s.Log.Trace(fmt.Sprintf("Target %s is a mount point. Checking if it is already mounted to source %s or %s", target, source, mapperSourcePath))

mountedDevicePath, _, err := mountutils.GetDeviceNameFromMount(s.NodeStorage.Interface, target)
if err != nil {
return fmt.Errorf("[PathExists] could not check if target directory %s exists: %w", target, err)
return fmt.Errorf("failed to find the device mounted at %s: %w", target, err)
}
if !exists {
s.Log.Debug(fmt.Sprintf("Creating target directory %s", target))
if err := os.MkdirAll(target, os.FileMode(0755)); err != nil {
return fmt.Errorf("[MkdirAll] could not create target directory %s: %w", target, err)
}
}
s.Log.Trace("-----------------== stop MkdirAll ==-----------------")
s.Log.Trace(fmt.Sprintf("Found device mounted at %s: %s", target, mountedDevicePath))

isMountPoint, err := s.NodeStorage.IsMountPoint(target)
if err != nil {
return fmt.Errorf("[s.NodeStorage.IsMountPoint] unable to determine mount status of %s: %w", target, err)
if mountedDevicePath != source && mountedDevicePath != mapperSourcePath {
return fmt.Errorf("target %s is a mount point and is not mounted to source %s or %s", target, source, mapperSourcePath)
}

s.Log.Trace("≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈ isMountPoint ≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈")
s.Log.Trace(fmt.Sprintf("%t", isMountPoint))
s.Log.Trace("≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈ isMountPoint ≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈")
s.Log.Trace(fmt.Sprintf("Target %s is a mount point and already mounted to source %s. Skipping FormatAndMount without any checks", target, source))
return nil
}

if isMountPoint {
mapperSourcePath := toMapperPath(devSourcePath)
s.Log.Trace(fmt.Sprintf("Target %s is a mount point. Checking if it is already mounted to source %s or %s", target, devSourcePath, mapperSourcePath))
s.Log.Trace("-----------------== start FormatAndMount ==---------------")

mountedDevicePath, _, err := mountutils.GetDeviceNameFromMount(s.NodeStorage.Interface, target)
if err != nil {
return fmt.Errorf("failed to find the device mounted at %s: %w", target, err)
}
s.Log.Trace(fmt.Sprintf("Found device mounted at %s: %s", target, mountedDevicePath))
if lvmType == internal.LVMTypeThin {
s.Log.Trace(fmt.Sprintf("LVM type is Thin. Thin pool name: %s", lvmThinPoolName))
}
err = s.NodeStorage.FormatAndMount(source, target, fsType, mountOpts)
if err != nil {
return fmt.Errorf("failed to FormatAndMount : %w", err)
}
s.Log.Trace("-----------------== stop FormatAndMount ==---------------")

if mountedDevicePath != devSourcePath && mountedDevicePath != mapperSourcePath {
return fmt.Errorf("target %s is a mount point and is not mounted to source %s or %s", target, devSourcePath, mapperSourcePath)
}
s.Log.Trace("-----------------== stop NodeStageVolumeFS ==---------------")
return nil
}

s.Log.Trace(fmt.Sprintf("Target %s is a mount point and already mounted to source %s. Skipping FormatAndMount without any checks", target, devSourcePath))
return nil
}
func (s *Store) NodePublishVolumeBlock(source, target string, mountOpts []string) error {
s.Log.Trace(" ----== Start NodePublishVolumeBlock ==---- ")

s.Log.Trace("-----------------== start FormatAndMount ==---------------")
s.Log.Trace("≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈ Mount options ≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈")
s.Log.Trace(fmt.Sprintf("[NodePublishVolumeBlock] params source=%s target=%s mountOptions=%v", source, target, mountOpts))
s.Log.Trace("≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈ Mount options ≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈")

if lvmType == internal.LVMTypeThin {
s.Log.Trace(fmt.Sprintf("LVM type is Thin. Thin pool name: %s", lvmThinPoolName))
}
err = s.NodeStorage.FormatAndMount(devSourcePath, target, fsType, mntOpts)
if err != nil {
return fmt.Errorf("failed to FormatAndMount : %w", err)
info, err := os.Stat(source)
if err != nil {
return fmt.Errorf("failed to stat source device: %w", err)
}

if (info.Mode() & os.ModeDevice) != os.ModeDevice {
return fmt.Errorf("[NodePublishVolumeBlock] path %s is not a device", source)
}

s.Log.Trace("≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈ MODE SOURCE ≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈")
s.Log.Trace(info.Mode().String())
s.Log.Trace("≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈ MODE SOURCE ≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈")

s.Log.Trace("-----------------== start Create File ==---------------")
f, err := os.OpenFile(target, os.O_CREATE, os.FileMode(0644))
if err != nil {
if !os.IsExist(err) {
return fmt.Errorf("[NodePublishVolumeBlock] could not create bind target for block volume %s, %w", target, err)
}
s.Log.Trace("-----------------== stop FormatAndMount ==---------------")
return nil
} else {
_ = f.Close()
}
s.Log.Trace("-----------------== stop Create File ==---------------")
s.Log.Trace("-----------------== start Mount ==---------------")
err = s.NodeStorage.Mount(source, target, "", mountOpts)
if err != nil {
s.Log.Error(err, "[NodePublishVolumeBlock] mount error :")
return err
}
s.Log.Trace("-----------------== stop Mount ==---------------")
s.Log.Trace("-----------------== stop NodePublishVolumeBlock ==---------------")
return nil
}

if isBlock {
s.Log.Trace("≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈ BLOCK MOUNT ≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈")
s.Log.Trace("-----------------== start Create File ==---------------")
f, err := os.OpenFile(target, os.O_CREATE, os.FileMode(0644))
func (s *Store) NodePublishVolumeFS(source, target, fsType string, mountOpts []string) error {
s.Log.Info(" ----== Start NodePublishVolumeFS ==---- ")
s.Log.Trace(fmt.Sprintf("[NodePublishVolumeFS] params source=%q target=%q mountOptions=%v", source, target, mountOpts))
isMountPoint := false
exists, err := s.PathExists(target)
if err != nil {
return fmt.Errorf("[NodePublishVolumeFS] could not check if target file %s exists: %w", target, err)
}

if exists {
s.Log.Trace(fmt.Sprintf("[NodePublishVolumeFS] target file %s already exists", target))
isMountPoint, err = s.NodeStorage.IsMountPoint(target)
if err != nil {
if !os.IsExist(err) {
return fmt.Errorf("could not create bind target for block volume %s, %w", target, err)
}
} else {
_ = f.Close()
return fmt.Errorf("[NodePublishVolumeFS] could not check if target file %s is a mount point: %w", target, err)
}
} else {
s.Log.Trace(fmt.Sprintf("[NodePublishVolumeFS] creating target file %q", target))
if err := os.MkdirAll(target, os.FileMode(0755)); err != nil {
return fmt.Errorf("[NodePublishVolumeFS] could not create target file %q: %w", target, err)
}
s.Log.Trace("-----------------== stop Create File ==---------------")
s.Log.Trace("-----------------== start Mount ==---------------")
err = s.NodeStorage.Mount(devSourcePath, target, fsType, mntOpts)
}

if isMountPoint {
s.Log.Trace(fmt.Sprintf("[NodePublishVolumeFS] target directory %q is a mount point. Check mount", target))
err := checkMount(s, source, target, mountOpts)
if err != nil {
s.Log.Error(err, "block mount error :")
return err
return fmt.Errorf("[NodePublishVolumeFS] failed to check mount info for %q: %w", target, err)
}
s.Log.Trace("-----------------== stop Mount ==---------------")
s.Log.Trace("≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈ BLOCK MOUNT ≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈")
s.Log.Trace(fmt.Sprintf("[NodePublishVolumeFS] target directory %q is a mount point and already mounted to source %q", target, source))
return nil
}
s.Log.Info("-----------------== Final ==---------------")

err = s.NodeStorage.Interface.Mount(source, target, fsType, mountOpts)
if err != nil {
return fmt.Errorf("[NodePublishVolumeFS] failed to bind mount %q to %q with mount options %v: %w", source, target, mountOpts, err)
}

s.Log.Trace("-----------------== stop NodePublishVolumeFS ==---------------")
return nil
}

Expand Down Expand Up @@ -218,47 +272,6 @@ func (s *Store) NeedResize(devicePath string, deviceMountPath string) (bool, err
return mountutils.NewResizeFs(s.NodeStorage.Exec).NeedResize(devicePath, deviceMountPath)
}

func (s *Store) BindMount(source, target, fsType string, mountOpts []string) error {
s.Log.Info(" ----== Bind Mount ==---- ")
s.Log.Trace(fmt.Sprintf("[BindMount] params source=%q target=%q mountOptions=%v", source, target, mountOpts))
isMountPoint := false
exists, err := s.PathExists(target)
if err != nil {
return fmt.Errorf("[BindMount] could not check if target file %s exists: %w", target, err)
}

if exists {
s.Log.Trace(fmt.Sprintf("[BindMount] target file %s already exists", target))
isMountPoint, err = s.NodeStorage.IsMountPoint(target)
if err != nil {
return fmt.Errorf("[BindMount] could not check if target file %s is a mount point: %w", target, err)
}
} else {
s.Log.Trace(fmt.Sprintf("[BindMount] creating target file %q", target))
if err := os.MkdirAll(target, os.FileMode(0755)); err != nil {
return fmt.Errorf("[BindMount] could not create target file %q: %w", target, err)
}
}

if isMountPoint {
s.Log.Trace(fmt.Sprintf("[BindMount] target directory %q is a mount point. Check mount", target))
err := checkMount(s, source, target, mountOpts)
if err != nil {
return fmt.Errorf("[BindMount] failed to check mount info for %q: %w", target, err)
}
s.Log.Trace(fmt.Sprintf("[BindMount] target directory %q is a mount point and already mounted to source %q", target, source))
return nil
}
// if err != nil {

err = s.NodeStorage.Interface.Mount(source, target, fsType, mountOpts)
if err != nil {
return fmt.Errorf("[BindMount] failed to bind mount %q to %q with mount options %v: %w", source, target, mountOpts, err)
}

return nil
}

func toMapperPath(devPath string) string {
if !strings.HasPrefix(devPath, "/dev/") {
return ""
Expand Down

0 comments on commit 56af844

Please sign in to comment.