Skip to content

Commit 378b953

Browse files
balamuruganabalamurugana
authored
Make Node{Stage,Publish}Volume RPCs idempotent on kubelet restarts (#752)
When kubelet is restarted NodeStageVolume/NodePublishVolume RPCs are called with existing staging-target-path and target-path. Previously respective RPCs return errors stating staging-target-path/target-path are already mounted. This PR fixes the issue. Signed-off-by: Bala.FA <bala@minio.io>
1 parent 6db246b commit 378b953

13 files changed

+93
-51
lines changed

‎pkg/csi/node/fake.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -34,8 +34,8 @@ func createFakeServer() *Server {
3434
rack: "test-rack",
3535
zone: "test-zone",
3636
region: "test-region",
37-
getMounts: func() (map[string]utils.StringSet, error) {
38-
return map[string]utils.StringSet{consts.MountRootDir: nil}, nil
37+
getMounts: func() (map[string]utils.StringSet, map[string]utils.StringSet, error) {
38+
return map[string]utils.StringSet{consts.MountRootDir: nil}, map[string]utils.StringSet{consts.MountRootDir: nil}, nil
3939
},
4040
getDeviceByFSUUID: func(fsuuid string) (string, error) { return "", nil },
4141
bindMount: func(source, target string, readOnly bool) error { return nil },

‎pkg/csi/node/publish_unpublish.go

+11-5
Original file line numberDiff line numberDiff line change
@@ -120,12 +120,14 @@ func (server *Server) NodePublishVolume(ctx context.Context, req *csi.NodePublis
120120
}
121121
}
122122

123-
mountPointMap, err := server.getMounts()
123+
mountPointMap, _, err := server.getMounts()
124124
if err != nil {
125125
klog.ErrorS(err, "unable to get mounts")
126126
return nil, status.Error(codes.Internal, err.Error())
127127
}
128-
if _, found := mountPointMap[req.GetStagingTargetPath()]; !found {
128+
129+
stagingTargetPathDevices, found := mountPointMap[req.GetStagingTargetPath()]
130+
if !found {
129131
klog.Errorf("stagingPath %v is not mounted", req.GetStagingTargetPath())
130132
return nil, status.Error(codes.Internal, fmt.Sprintf("stagingPath %v is not mounted", req.GetStagingTargetPath()))
131133
}
@@ -140,9 +142,13 @@ func (server *Server) NodePublishVolume(ctx context.Context, req *csi.NodePublis
140142
return nil, status.Errorf(codes.Internal, "unable to create target path: %v", err)
141143
}
142144

143-
if err := server.bindMount(req.GetStagingTargetPath(), req.GetTargetPath(), req.GetReadonly()); err != nil {
144-
klog.ErrorS(err, "unable to bind mount staging target path to target path", "StagingTargetPath", req.GetStagingTargetPath(), "TargetPath", req.GetTargetPath())
145-
return nil, status.Errorf(codes.Internal, "unable to bind mount staging target path to target path; %v", err)
145+
if targetPathDevices, found := mountPointMap[req.GetTargetPath()]; found && targetPathDevices.Equal(stagingTargetPathDevices) {
146+
klog.V(5).InfoS("stagingTargetPath is already bind-mounted to targetPath", "stagingTargetPath", req.GetStagingTargetPath(), "targetPath", req.GetTargetPath())
147+
} else {
148+
if err := server.bindMount(req.GetStagingTargetPath(), req.GetTargetPath(), req.GetReadonly()); err != nil {
149+
klog.ErrorS(err, "unable to bind mount staging target path to target path", "StagingTargetPath", req.GetStagingTargetPath(), "TargetPath", req.GetTargetPath())
150+
return nil, status.Errorf(codes.Internal, "unable to bind mount staging target path to target path; %v", err)
151+
}
146152
}
147153

148154
volume.Status.TargetPath = req.GetTargetPath()

‎pkg/csi/node/publish_unpublish_test.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -110,8 +110,8 @@ func TestPublishUnpublishVolume(t *testing.T) {
110110
ns := createFakeServer()
111111

112112
// Publish volume test
113-
ns.getMounts = func() (map[string]utils.StringSet, error) {
114-
return map[string]utils.StringSet{testStagingPath: nil}, nil
113+
ns.getMounts = func() (map[string]utils.StringSet, map[string]utils.StringSet, error) {
114+
return map[string]utils.StringSet{testStagingPath: nil}, map[string]utils.StringSet{testStagingPath: nil}, nil
115115
}
116116
_, err := ns.NodePublishVolume(ctx, &publishVolumeRequest)
117117
if err != nil {

‎pkg/csi/node/server.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ type Server struct {
4242
zone string
4343
region string
4444

45-
getMounts func() (map[string]utils.StringSet, error)
45+
getMounts func() (mountMap, rootMap map[string]utils.StringSet, err error)
4646
getDeviceByFSUUID func(fsuuid string) (string, error)
4747
bindMount func(source, target string, readOnly bool) error
4848
unmount func(target string) error
@@ -59,8 +59,8 @@ func newServer(identity string, nodeID directpvtypes.NodeID, rack, zone, region
5959
zone: zone,
6060
region: region,
6161

62-
getMounts: func() (mountMap map[string]utils.StringSet, err error) {
63-
mountMap, _, _, err = sys.GetMounts(false)
62+
getMounts: func() (mountMap, rootMap map[string]utils.StringSet, err error) {
63+
mountMap, _, _, rootMap, err = sys.GetMounts(false)
6464
return
6565
},
6666
getDeviceByFSUUID: sys.GetDeviceByFSUUID,

‎pkg/csi/node/stage_unstage.go

+5
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import (
2323
"github.com/minio/directpv/pkg/client"
2424
"github.com/minio/directpv/pkg/drive"
2525
"github.com/minio/directpv/pkg/types"
26+
"github.com/minio/directpv/pkg/utils"
2627
"google.golang.org/grpc/codes"
2728
"google.golang.org/grpc/status"
2829
apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -61,6 +62,10 @@ func (server *Server) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol
6162
server.mkdir,
6263
server.setQuota,
6364
server.bindMount,
65+
func() (rootMap map[string]utils.StringSet, err error) {
66+
_, rootMap, err = server.getMounts()
67+
return
68+
},
6469
)
6570
if err != nil {
6671
return nil, status.Error(code, err.Error())

‎pkg/csi/node/stage_unstage_test.go

+4-4
Original file line numberDiff line numberDiff line change
@@ -76,8 +76,8 @@ func TestNodeStageVolume(t *testing.T) {
7676
client.SetVolumeInterface(clientset.DirectpvLatest().DirectPVVolumes())
7777

7878
nodeServer := createFakeServer()
79-
nodeServer.getMounts = func() (map[string]utils.StringSet, error) {
80-
return testCase.mountInfo, nil
79+
nodeServer.getMounts = func() (map[string]utils.StringSet, map[string]utils.StringSet, error) {
80+
return testCase.mountInfo, testCase.mountInfo, nil
8181
}
8282
nodeServer.bindMount = func(source, stagingTargetPath string, readOnly bool) error {
8383
if _, found := testCase.mountInfo[source]; !found {
@@ -147,8 +147,8 @@ func TestStageUnstageVolume(t *testing.T) {
147147
ctx := context.TODO()
148148
ns := createFakeServer()
149149
dataPath := path.Join(consts.MountRootDir, testDriveName, ".FSUUID."+testDriveName, testVolumeName50MB)
150-
ns.getMounts = func() (map[string]utils.StringSet, error) {
151-
return map[string]utils.StringSet{consts.MountRootDir: nil}, nil
150+
ns.getMounts = func() (map[string]utils.StringSet, map[string]utils.StringSet, error) {
151+
return map[string]utils.StringSet{consts.MountRootDir: nil}, map[string]utils.StringSet{consts.MountRootDir: nil}, nil
152152
}
153153

154154
// Stage Volume test

‎pkg/device/probe_linux.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ func probe() (devices []Device, err error) {
8888
return nil, err
8989
}
9090

91-
_, deviceMountMap, majorMinorMap, err := sys.GetMounts(true)
91+
_, deviceMountMap, majorMinorMap, _, err := sys.GetMounts(true)
9292
if err != nil {
9393
return nil, err
9494
}
@@ -115,7 +115,7 @@ func probe() (devices []Device, err error) {
115115
}
116116

117117
func probeDevices(majorMinor ...string) (devices []Device, err error) {
118-
_, deviceMountMap, majorMinorMap, err := sys.GetMounts(true)
118+
_, deviceMountMap, majorMinorMap, _, err := sys.GetMounts(true)
119119
if err != nil {
120120
return nil, err
121121
}

‎pkg/drive/event.go

+28-5
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,24 @@ func SetIOError(ctx context.Context, driveID directpvtypes.DriveID) error {
6363
return retry.RetryOnConflict(retry.DefaultRetry, updateFunc)
6464
}
6565

66+
func stageVolumeMount(
67+
volumeName, volumeDir, stagingTargetPath string,
68+
getMounts func() (map[string]utils.StringSet, error),
69+
bindMount func(volumeDir, stagingTargetPath string, readOnly bool) error,
70+
) error {
71+
rootMountPointMap, err := getMounts()
72+
if err != nil {
73+
return err
74+
}
75+
76+
if mountPoints, found := rootMountPointMap["/"+volumeName]; found && mountPoints.Exist(stagingTargetPath) {
77+
klog.V(5).InfoS("volumeDir is already bind-mounted on stagingTargetPath", "volumeDir", volumeDir, "stagingTargetPath", stagingTargetPath)
78+
return nil
79+
}
80+
81+
return bindMount(volumeDir, stagingTargetPath, false)
82+
}
83+
6684
// StageVolume creates and mounts staging target path of the volume to the drive.
6785
func StageVolume(
6886
ctx context.Context,
@@ -72,6 +90,7 @@ func StageVolume(
7290
mkdir func(volumeDir string) error,
7391
setQuota func(ctx context.Context, device, stagingTargetPath, volumeName string, quota xfs.Quota, update bool) error,
7492
bindMount func(volumeDir, stagingTargetPath string, readOnly bool) error,
93+
getMounts func() (map[string]utils.StringSet, error),
7594
) (codes.Code, error) {
7695
device, err := getDeviceByFSUUID(volume.Status.FSUUID)
7796
if err != nil {
@@ -114,7 +133,7 @@ func StageVolume(
114133
}
115134

116135
if stagingTargetPath != "" {
117-
if err := bindMount(volumeDir, stagingTargetPath, false); err != nil {
136+
if err := stageVolumeMount(volume.Name, volumeDir, stagingTargetPath, getMounts, bindMount); err != nil {
118137
return codes.Internal, fmt.Errorf("unable to bind mount volume directory to staging target path; %w", err)
119138
}
120139
}
@@ -133,7 +152,7 @@ func StageVolume(
133152

134153
type driveEventHandler struct {
135154
nodeID directpvtypes.NodeID
136-
getMounts func() (mountPointMap, deviceMap map[string]utils.StringSet, err error)
155+
getMounts func() (mountPointMap, deviceMap, rootMountPointMap map[string]utils.StringSet, err error)
137156
unmount func(target string) error
138157
mkdir func(path string) error
139158
bindMount func(source, target string, readOnly bool) error
@@ -145,8 +164,8 @@ type driveEventHandler struct {
145164
func newDriveEventHandler(nodeID directpvtypes.NodeID) *driveEventHandler {
146165
return &driveEventHandler{
147166
nodeID: nodeID,
148-
getMounts: func() (mountPointMap, deviceMap map[string]utils.StringSet, err error) {
149-
mountPointMap, deviceMap, _, err = sys.GetMounts(false)
167+
getMounts: func() (mountPointMap, deviceMap, rootMountPointMap map[string]utils.StringSet, err error) {
168+
mountPointMap, deviceMap, _, rootMountPointMap, err = sys.GetMounts(false)
150169
return
151170
},
152171
unmount: func(mountPoint string) error {
@@ -189,7 +208,7 @@ func (handler *driveEventHandler) ObjectType() runtime.Object {
189208
}
190209

191210
func (handler *driveEventHandler) unmountDrive(drive *types.Drive, skipDriveMount bool) error {
192-
mountPointMap, deviceMap, err := handler.getMounts()
211+
mountPointMap, deviceMap, _, err := handler.getMounts()
193212
if err != nil {
194213
return err
195214
}
@@ -280,6 +299,10 @@ func (handler *driveEventHandler) move(ctx context.Context, drive *types.Drive)
280299
handler.mkdir,
281300
handler.setQuota,
282301
handler.bindMount,
302+
func() (rootMap map[string]utils.StringSet, err error) {
303+
_, _, rootMap, err = handler.getMounts()
304+
return
305+
},
283306
)
284307
if err != nil && !errors.Is(err, os.ErrNotExist) {
285308
klog.ErrorS(err, "unable to stage volume after volume move",

‎pkg/initrequest/event.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ func newInitRequestEventHandler(ctx context.Context, nodeID directpvtypes.NodeID
8484
probeDevices: pkgdevice.Probe,
8585
getDevices: pkgdevice.ProbeDevices,
8686
getMounts: func() (deviceMap, majorMinorMap map[string]utils.StringSet, err error) {
87-
if _, deviceMap, majorMinorMap, err = sys.GetMounts(true); err != nil {
87+
if _, deviceMap, majorMinorMap, _, err = sys.GetMounts(true); err != nil {
8888
err = fmt.Errorf("unable get mount points; %w", err)
8989
}
9090
return

‎pkg/sys/mount.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ func (e *ErrMountPointAlreadyMounted) Error() string {
3535
}
3636

3737
// GetMounts returns mount-point to devices and devices to mount-point maps.
38-
func GetMounts(includeMajorMinorMap bool) (mountPointMap, deviceMap, majorMinorMap map[string]utils.StringSet, err error) {
38+
func GetMounts(includeMajorMinorMap bool) (mountPointMap, deviceMap, majorMinorMap, rootMountPointMap map[string]utils.StringSet, err error) {
3939
return getMounts(includeMajorMinorMap)
4040
}
4141

‎pkg/sys/mount_linux.go

+17-24
Original file line numberDiff line numberDiff line change
@@ -32,19 +32,20 @@ import (
3232
"k8s.io/klog/v2"
3333
)
3434

35-
func parseProc1Mountinfo(r io.Reader) (mountPointMap, deviceMap, majorMinorMap map[string]utils.StringSet, err error) {
35+
func parseProc1Mountinfo(r io.Reader) (mountPointMap, deviceMap, majorMinorMap, rootMountPointMap map[string]utils.StringSet, err error) {
3636
reader := bufio.NewReader(r)
3737

3838
mountPointMap = make(map[string]utils.StringSet)
3939
deviceMap = make(map[string]utils.StringSet)
4040
majorMinorMap = make(map[string]utils.StringSet)
41+
rootMountPointMap = make(map[string]utils.StringSet)
4142
for {
4243
s, err := reader.ReadString('\n')
4344
if err != nil {
4445
if errors.Is(err, io.EOF) {
4546
break
4647
}
47-
return nil, nil, nil, err
48+
return nil, nil, nil, nil, err
4849
}
4950

5051
// Refer /proc/[pid]/mountinfo section in https://man7.org/linux/man-pages/man5/proc.5.html
@@ -64,6 +65,7 @@ func parseProc1Mountinfo(r io.Reader) (mountPointMap, deviceMap, majorMinorMap m
6465
}
6566

6667
majorMinor := tokens[2]
68+
root := tokens[3]
6769
mountPoint := tokens[4]
6870
device := tokens[i+1]
6971

@@ -81,6 +83,11 @@ func parseProc1Mountinfo(r io.Reader) (mountPointMap, deviceMap, majorMinorMap m
8183
majorMinorMap[majorMinor] = make(utils.StringSet)
8284
}
8385
majorMinorMap[majorMinor].Set(device)
86+
87+
if _, found := rootMountPointMap[root]; !found {
88+
rootMountPointMap[root] = make(utils.StringSet)
89+
}
90+
rootMountPointMap[root].Set(mountPoint)
8491
}
8592

8693
return
@@ -94,7 +101,7 @@ func getMajorMinor(device string) (majorMinor string, err error) {
94101
return
95102
}
96103

97-
func parseProcMounts(r io.Reader, includeMajorMinorMap bool) (mountPointMap, deviceMap, majorMinorMap map[string]utils.StringSet, err error) {
104+
func parseProcMounts(r io.Reader, includeMajorMinorMap bool) (mountPointMap, deviceMap, majorMinorMap, rootMountPointMap map[string]utils.StringSet, err error) {
98105
reader := bufio.NewReader(r)
99106

100107
mountPointMap = make(map[string]utils.StringSet)
@@ -105,7 +112,7 @@ func parseProcMounts(r io.Reader, includeMajorMinorMap bool) (mountPointMap, dev
105112
if errors.Is(err, io.EOF) {
106113
break
107114
}
108-
return nil, nil, nil, err
115+
return nil, nil, nil, nil, err
109116
}
110117

111118
// Refer /proc/mounts section in https://man7.org/linux/man-pages/man5/proc.5.html
@@ -142,7 +149,7 @@ func parseProcMounts(r io.Reader, includeMajorMinorMap bool) (mountPointMap, dev
142149

143150
majorMinor, err := getMajorMinor(device)
144151
if err != nil {
145-
return nil, nil, nil, err
152+
return nil, nil, nil, nil, err
146153
}
147154

148155
if _, found := majorMinorMap[device]; !found {
@@ -154,19 +161,19 @@ func parseProcMounts(r io.Reader, includeMajorMinorMap bool) (mountPointMap, dev
154161
return
155162
}
156163

157-
func getMounts(includeMajorMinorMap bool) (mountPointMap, deviceMap, majorMinorMap map[string]utils.StringSet, err error) {
164+
func getMounts(includeMajorMinorMap bool) (mountPointMap, deviceMap, majorMinorMap, rootMountPointMap map[string]utils.StringSet, err error) {
158165
file, err := os.Open("/proc/1/mountinfo")
159166
if err != nil {
160167
if !errors.Is(err, os.ErrNotExist) {
161-
return nil, nil, nil, err
168+
return nil, nil, nil, nil, err
162169
}
163170
} else {
164171
defer file.Close()
165172
return parseProc1Mountinfo(file)
166173
}
167174

168175
if file, err = os.Open("/proc/mounts"); err != nil {
169-
return nil, nil, nil, err
176+
return nil, nil, nil, nil, err
170177
}
171178

172179
defer file.Close()
@@ -197,7 +204,7 @@ var mountFlagMap = map[string]uintptr{
197204
}
198205

199206
func mount(device, target, fsType string, flags []string, superBlockFlags string) error {
200-
mountPointMap, _, _, err := getMounts(false)
207+
mountPointMap, _, _, _, err := getMounts(false)
201208
if err != nil {
202209
return err
203210
}
@@ -224,20 +231,6 @@ func mount(device, target, fsType string, flags []string, superBlockFlags string
224231
}
225232

226233
func bindMount(source, target, fsType string, recursive, readOnly bool, superBlockFlags string) error {
227-
mountPointMap, _, _, err := getMounts(false)
228-
if err != nil {
229-
return err
230-
}
231-
232-
if devices, found := mountPointMap[target]; found {
233-
if devices.Exist(source) {
234-
klog.V(5).InfoS("source is already mounted on target", "source", source, "target", target, "fsType", fsType, "recursive", recursive, "readOnly", readOnly, "superBlockFlags", superBlockFlags)
235-
return nil
236-
}
237-
238-
return &ErrMountPointAlreadyMounted{MountPoint: target, Devices: devices.ToSlice()}
239-
}
240-
241234
flags := mountFlagMap["bind"]
242235
if recursive {
243236
flags |= mountFlagMap["recursive"]
@@ -250,7 +243,7 @@ func bindMount(source, target, fsType string, recursive, readOnly bool, superBlo
250243
}
251244

252245
func unmount(target string, force, detach, expire bool) error {
253-
mountPointMap, _, _, err := getMounts(false)
246+
mountPointMap, _, _, _, err := getMounts(false)
254247
if err != nil {
255248
return err
256249
}

‎pkg/sys/mount_other.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,8 @@ import (
2525
"github.com/minio/directpv/pkg/utils"
2626
)
2727

28-
func getMounts(includeMajorMinorMap bool) (mountPointMap, deviceMap, majorMinorMap map[string]utils.StringSet, err error) {
29-
return nil, nil, nil, fmt.Errorf("unsupported operating system %v", runtime.GOOS)
28+
func getMounts(includeMajorMinorMap bool) (mountPointMap, deviceMap, majorMinorMap, rootMountPointMap map[string]utils.StringSet, err error) {
29+
return nil, nil, nil, nil, fmt.Errorf("unsupported operating system %v", runtime.GOOS)
3030
}
3131

3232
func mount(device, target, fsType string, flags []string, superBlockFlags string) error {

‎pkg/utils/utils.go

+15
Original file line numberDiff line numberDiff line change
@@ -134,6 +134,21 @@ func (set StringSet) ToSlice() (values []string) {
134134
return
135135
}
136136

137+
// Equal checks whether given StringSet is same or not.
138+
func (set StringSet) Equal(set2 StringSet) (found bool) {
139+
if len(set) != len(set2) {
140+
return false
141+
}
142+
143+
for value := range set {
144+
if _, found := set2[value]; !found {
145+
return false
146+
}
147+
}
148+
149+
return true
150+
}
151+
137152
// Eprintf prints the message to the stdout and stderr based on inputs
138153
func Eprintf(quiet, asErr bool, format string, a ...any) {
139154
if quiet {

0 commit comments

Comments
 (0)