Skip to content

Commit b6c2f59

Browse files
YashasG98YashwantGohokar
YashasG98
authored and
YashwantGohokar
committed
Fix CSI mount detection logic
1 parent 6154210 commit b6c2f59

File tree

4 files changed

+167
-3
lines changed

4 files changed

+167
-3
lines changed

‎pkg/csi/driver/bv_node.go‎

Lines changed: 31 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -147,7 +147,7 @@ func (d BlockVolumeNodeDriver) NodeStageVolume(ctx context.Context, req *csi.Nod
147147
defer d.volumeLocks.Release(req.VolumeId)
148148

149149
if !isRawBlockVolume {
150-
isMounted, oErr := mountHandler.DeviceOpened(devicePath)
150+
isMounted, oErr := mountHandler.IsMounted(devicePath, req.StagingTargetPath)
151151
if oErr != nil {
152152
logger.With(zap.Error(oErr)).Error("getting error to get the details about volume is already mounted or not.")
153153
return nil, status.Error(codes.Internal, oErr.Error())
@@ -197,6 +197,10 @@ func (d BlockVolumeNodeDriver) NodeStageVolume(ctx context.Context, req *csi.Nod
197197
devicePath, err = disk.WaitForDevicePathToExist(ctx, scsiInfo, logger)
198198
if err != nil {
199199
logger.With(zap.Error(err)).Error("Failed to get /dev/disk/by-path device path for iscsi volume.")
200+
err = mountHandler.ISCSILogoutOnFailure()
201+
if err != nil {
202+
return nil, status.Error(codes.Internal, "Failed to iscsi logout after timeout on waiting for device path to exist")
203+
}
200204
return nil, status.Error(codes.InvalidArgument, "Failed to get device path for iscsi volume")
201205
}
202206
}
@@ -215,12 +219,20 @@ func (d BlockVolumeNodeDriver) NodeStageVolume(ctx context.Context, req *csi.Nod
215219
err := csi_util.CreateFilePath(logger, stagingTargetFilePath)
216220
if err != nil {
217221
logger.With(zap.Error(err)).Error("failed to create the stagingTargetFile.")
222+
err = mountHandler.ISCSILogoutOnFailure()
223+
if err != nil {
224+
return nil, status.Error(codes.Internal, "Failed to iscsi logout after mount failure")
225+
}
218226
return nil, status.Error(codes.Internal, err.Error())
219227
}
220228
options := []string{"bind"} // Append the "bind" option if it is a raw block volume
221229
err = mountHandler.Mount(devicePath, stagingTargetFilePath, "", options)
222230
if err != nil {
223231
logger.With(zap.Error(err)).Error("failed to bind mount raw block volume to stagingTargetFile")
232+
err = mountHandler.ISCSILogoutOnFailure()
233+
if err != nil {
234+
return nil, status.Error(codes.Internal, "Failed to iscsi logout after mount failure")
235+
}
224236
return nil, status.Error(codes.Internal, err.Error())
225237
}
226238
return &csi.NodeStageVolumeResponse{}, nil
@@ -238,6 +250,10 @@ func (d BlockVolumeNodeDriver) NodeStageVolume(ctx context.Context, req *csi.Nod
238250
exists = false
239251
} else {
240252
logger.With(zap.Error(err)).Errorf("failed to check if stagingTargetPath %q exists", req.StagingTargetPath)
253+
err = mountHandler.ISCSILogoutOnFailure()
254+
if err != nil {
255+
return nil, status.Error(codes.Internal, "Failed to iscsi logout after failure to check if staging path exists")
256+
}
241257
message := fmt.Sprintf("failed to check if stagingTargetPath %q exists", req.StagingTargetPath)
242258
return nil, status.Error(codes.Internal, message)
243259
}
@@ -249,6 +265,10 @@ func (d BlockVolumeNodeDriver) NodeStageVolume(ctx context.Context, req *csi.Nod
249265
if !exists {
250266
if err := os.MkdirAll(req.StagingTargetPath, 0750); err != nil {
251267
logger.With(zap.Error(err)).Error("Failed to create StagingTargetPath directory")
268+
err = mountHandler.ISCSILogoutOnFailure()
269+
if err != nil {
270+
return nil, status.Error(codes.Internal, "Failed to iscsi logout after failure to create StagingTargetPath directory")
271+
}
252272
return nil, status.Error(codes.Internal, "Failed to create StagingTargetPath directory")
253273
}
254274
}
@@ -270,6 +290,10 @@ func (d BlockVolumeNodeDriver) NodeStageVolume(ctx context.Context, req *csi.Nod
270290
if existingFs != "" && existingFs != fsType {
271291
returnError := fmt.Sprintf("FS Type mismatch detected. The existing fs type on the volume: %q doesn't match the requested fs type: %q. Please change fs type in PV to match the existing fs type.", existingFs, fsType)
272292
logger.Error(returnError)
293+
err = mountHandler.ISCSILogoutOnFailure()
294+
if err != nil {
295+
return nil, status.Error(codes.Internal, "Failed to iscsi logout after failure due to FS Type mismatch")
296+
}
273297
return nil, status.Error(codes.Internal, returnError)
274298
}
275299

@@ -278,6 +302,10 @@ func (d BlockVolumeNodeDriver) NodeStageVolume(ctx context.Context, req *csi.Nod
278302
err = mountHandler.FormatAndMount(devicePath, req.StagingTargetPath, fsType, options)
279303
if err != nil {
280304
logger.With(zap.Error(err)).Error("failed to format and mount volume to staging path.")
305+
err = mountHandler.ISCSILogoutOnFailure()
306+
if err != nil {
307+
return nil, status.Error(codes.Internal, "Failed to iscsi logout after mount failure")
308+
}
281309
return nil, status.Error(codes.Internal, err.Error())
282310
}
283311
logger.With("devicePath", devicePath, "fsType", fsType, "attachmentType", attachment).
@@ -379,10 +407,10 @@ func (d BlockVolumeNodeDriver) NodeUnstageVolume(ctx context.Context, req *csi.N
379407
if !isRawBlockVolume {
380408
isMounted, oErr := mountHandler.DeviceOpened(devicePath)
381409
if oErr != nil {
382-
logger.With(zap.Error(oErr)).Error("getting error to get the details about volume is already mounted or not.")
410+
logger.With(zap.Error(oErr)).Error("getting error to get the details about volume is already unmounted or not.")
383411
return nil, status.Error(codes.Internal, oErr.Error())
384412
} else if !isMounted {
385-
logger.Info("volume is already unmounted on the staging path.")
413+
logger.Info("volume is already unmounted from the staging path.")
386414
return &csi.NodeUnstageVolumeResponse{}, nil
387415
}
388416
}

‎pkg/util/disk/iscsi.go‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,8 @@ import (
2929

3030
"github.com/oracle/oci-go-sdk/v65/core"
3131
"go.uber.org/zap"
32+
"google.golang.org/grpc/codes"
33+
"google.golang.org/grpc/status"
3234
"k8s.io/apimachinery/pkg/util/wait"
3335
"k8s.io/mount-utils"
3436
"k8s.io/utils/exec"
@@ -87,6 +89,9 @@ type Interface interface {
8789
Logout() error
8890

8991
DeviceOpened(pathname string) (bool, error)
92+
93+
IsMounted(devicePath string, targetPath string) (bool, error)
94+
9095
// updates the queue depth for iSCSI target
9196
UpdateQueueDepth() error
9297

@@ -110,6 +115,8 @@ type Interface interface {
110115
GetDiskFormat(devicePath string) (string, error)
111116

112117
WaitForPathToExist(path string, maxRetries int) bool
118+
119+
ISCSILogoutOnFailure() error
113120
}
114121

115122
// iSCSIMounter implements Interface.
@@ -483,6 +490,49 @@ func (c *iSCSIMounter) DeviceOpened(pathname string) (bool, error) {
483490
return deviceOpened(pathname, c.logger)
484491
}
485492

493+
func (c *iSCSIMounter) IsMounted(devicePath string, targetPath string) (bool, error) {
494+
var diskByPath string
495+
notMnt, err := c.mounter.IsLikelyNotMountPoint(targetPath)
496+
if err != nil {
497+
if os.IsNotExist(err){
498+
return false, nil
499+
}
500+
return false, fmt.Errorf("failed to check if %s is a mount point: %v", targetPath, err)
501+
}
502+
if notMnt {
503+
return false, nil
504+
}
505+
506+
diskByPath, err = GetIscsiDevicePath(c.disk)
507+
if err != nil {
508+
if strings.Contains(err.Error(), "No such file or directory") {
509+
return false, fmt.Errorf("ISCSI login not complete for volume but staging path is a mount point, mapped to wrong device")
510+
} else {
511+
return false, fmt.Errorf("failed to find /dev/disk/by-path path for volume: %v", c.disk)
512+
}
513+
}
514+
515+
resolvedDevicePath, err := filepath.EvalSymlinks(diskByPath)
516+
if err != nil {
517+
return false, fmt.Errorf("failed to resolve symlink for /dev/disk/by-path path %s: %v", diskByPath, err)
518+
}
519+
520+
mounts, err := c.mounter.List()
521+
if err != nil {
522+
return false, fmt.Errorf("could not list mount points: %v", err)
523+
}
524+
525+
for _, m := range mounts {
526+
if m.Path == targetPath {
527+
if m.Device == resolvedDevicePath {
528+
return true, nil
529+
}
530+
return false, fmt.Errorf("expected device %s but found %s mounted at %s", resolvedDevicePath, m.Device, targetPath)
531+
}
532+
}
533+
return false, nil
534+
}
535+
486536
func (c *iSCSIMounter) UnmountPath(path string) error {
487537
return UnmountPath(c.logger, path, c.mounter)
488538
}
@@ -500,6 +550,22 @@ func (c *iSCSIMounter) WaitForPathToExist(path string, maxRetries int) bool {
500550
return true
501551
}
502552

553+
func (c *iSCSIMounter) ISCSILogoutOnFailure() error {
554+
err := c.Logout()
555+
if err != nil {
556+
c.logger.With(zap.Error(err)).Error("failed to logout from the iSCSI target")
557+
return status.Error(codes.Internal, err.Error())
558+
}
559+
560+
err = c.RemoveFromDB()
561+
if err != nil {
562+
c.logger.With(zap.Error(err)).Error("failed to remove the iSCSI node record")
563+
return status.Error(codes.Internal, err.Error())
564+
}
565+
566+
return nil
567+
}
568+
503569
// getMountPointForPath returns the mount.MountPoint for a given path. If the
504570
// given path is not a mount point
505571
func getMountPointForPath(ml mount.Interface, path string) (mount.MountPoint, error) {

‎pkg/util/disk/iscsi_uhp.go‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -195,6 +195,37 @@ func (c *iSCSIUHPMounter) DeviceOpened(pathname string) (bool, error) {
195195
return deviceOpened(pathname, c.logger)
196196
}
197197

198+
func (c *iSCSIUHPMounter) IsMounted(devicePath string, targetPath string) (bool, error) {
199+
notMnt, err := c.mounter.IsLikelyNotMountPoint(targetPath)
200+
if err != nil {
201+
if os.IsNotExist(err){
202+
return false, nil
203+
}
204+
return false, fmt.Errorf("failed to check if %s is a mount point: %v", targetPath, err)
205+
}
206+
if notMnt {
207+
return false, nil
208+
}
209+
mounts, err := c.mounter.List()
210+
if err != nil {
211+
return false, fmt.Errorf("could not list mount points: %v", err)
212+
}
213+
214+
for _, m := range mounts {
215+
if m.Path == targetPath {
216+
if m.Device == devicePath {
217+
return true, nil
218+
}
219+
return false, fmt.Errorf("expected device %s but found %s mounted at %s", devicePath, m.Device, targetPath)
220+
}
221+
}
222+
return false, nil
223+
}
224+
225+
func (c *iSCSIUHPMounter) ISCSILogoutOnFailure() error {
226+
return nil
227+
}
228+
198229
func GetMultipathIscsiDevicePath(ctx context.Context, consistentDevicePath string, logger *zap.SugaredLogger) (string, error) {
199230
logger.With("consistentDevicePath", consistentDevicePath).Info("Getting friendly name of multipath device using consistent device path")
200231

‎pkg/util/disk/paravirtualized.go‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,13 +16,15 @@ package disk
1616

1717
import (
1818
"context"
19+
"fmt"
1920
"os"
2021
"time"
2122

2223
"github.com/oracle/oci-go-sdk/v65/core"
2324
"go.uber.org/zap"
2425
"k8s.io/mount-utils"
2526
"k8s.io/utils/exec"
27+
"path/filepath"
2628
)
2729

2830
// iSCSIMounter implements Interface.
@@ -99,6 +101,43 @@ func (c *pvMounter) DeviceOpened(pathname string) (bool, error) {
99101
return deviceOpened(pathname, c.logger)
100102
}
101103

104+
func (c *pvMounter) IsMounted(devicePath string, targetPath string) (bool, error) {
105+
notMnt, err := c.mounter.IsLikelyNotMountPoint(targetPath)
106+
if err != nil {
107+
if os.IsNotExist(err){
108+
return false, nil
109+
}
110+
return false, fmt.Errorf("failed to check if %s is a mount point: %v", targetPath, err)
111+
}
112+
if notMnt {
113+
return false, nil
114+
}
115+
116+
resolvedDevicePath, err := filepath.EvalSymlinks(devicePath)
117+
if err != nil {
118+
return false, fmt.Errorf("failed to resolve symlink for consistent device path %s: %v", devicePath, err)
119+
}
120+
121+
mounts, err := c.mounter.List()
122+
if err != nil {
123+
return false, fmt.Errorf("could not list mount points: %v", err)
124+
}
125+
126+
for _, m := range mounts {
127+
if m.Path == targetPath {
128+
if m.Device == resolvedDevicePath {
129+
return true, nil
130+
}
131+
return false, fmt.Errorf("expected device %s but found %s mounted at %s", resolvedDevicePath, m.Device, targetPath)
132+
}
133+
}
134+
return false, nil
135+
}
136+
137+
func (c *pvMounter) ISCSILogoutOnFailure() error {
138+
return nil
139+
}
140+
102141
func (c *pvMounter) UnmountPath(path string) error {
103142
return UnmountPath(c.logger, path, c.mounter)
104143
}

0 commit comments

Comments
 (0)