From 0c8c3de949fe2375a8e64e3a8c1d9ba4f4cc02ec Mon Sep 17 00:00:00 2001 From: Michael Mattsson Date: Fri, 24 Oct 2025 13:35:29 -0700 Subject: [PATCH 1/2] Switch from symlinks to block devices for raw block devices Signed-off-by: Michael Mattsson --- go.mod | 2 +- pkg/driver/node_server.go | 59 ++++++++++++++++++++++++++------------- 2 files changed, 40 insertions(+), 21 deletions(-) diff --git a/go.mod b/go.mod index d63bf46c..6b607952 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,7 @@ require ( github.com/stretchr/testify v1.10.0 golang.org/x/mod v0.24.0 golang.org/x/net v0.38.0 + golang.org/x/sys v0.33.0 google.golang.org/grpc v1.71.1 k8s.io/api v0.29.0 k8s.io/apimachinery v0.29.0 @@ -78,7 +79,6 @@ require ( go.uber.org/zap v1.27.0 // indirect golang.org/x/crypto v0.38.0 // indirect golang.org/x/oauth2 v0.25.0 // indirect - golang.org/x/sys v0.33.0 // indirect golang.org/x/term v0.32.0 // indirect golang.org/x/text v0.25.0 // indirect golang.org/x/time v0.5.0 // indirect diff --git a/pkg/driver/node_server.go b/pkg/driver/node_server.go index 0f69493d..1ca8876c 100644 --- a/pkg/driver/node_server.go +++ b/pkg/driver/node_server.go @@ -18,6 +18,7 @@ import ( "github.com/container-storage-interface/spec/lib/go/csi" "golang.org/x/net/context" + "golang.org/x/sys/unix" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "k8s.io/apimachinery/pkg/api/resource" @@ -1029,31 +1030,49 @@ func (driver *Driver) nodePublishVolume( // If Block, then stage the volume for raw block device access if volAccessType.String() == model.BlockType.String() { - log.Tracef("Publishing the volume for raw block access (create symlink), devicePath: %s, targetPath: %v", + log.Tracef("Publishing the volume for raw block access (create block device), devicePath: %s, targetPath: %v", stagingDev.Device.AltFullPathName, targetPath) - // Check if target path symlink to the device already exists - exists, symlink, _ := util.IsFileSymlink(targetPath) - if symlink { - errMsg := fmt.Sprintf("Target path %s already published as symlink to the device %s", targetPath, stagingDev.Device.AltFullPathName) + // Check if target path block device to the mpath device already exists + exists, blockDevice, _ := util.IsFileBlockDevice(targetPath) + if blockDevice { + errMsg := fmt.Sprintf("Target path %s already published as block device to the mpath device %s", targetPath, stagingDev.Device.AltFullPathName) log.Error("Error: ", errMsg) return status.Error(codes.Internal, errMsg) } if exists { // Remove the target path before creating the symlink - log.Tracef("Removing the target path %s before creating symlink to the device", targetPath) + log.Tracef("Removing the target block device %s before creating block device to the mpath device", targetPath) if err := util.FileDelete(targetPath); err != nil { return status.Error(codes.Internal, - fmt.Sprintf("Error removing the target path %s before creating symlink to the device, err: %s", + fmt.Sprintf("Error removing the target block device %s before creating block device to the mpath device, err: %s", targetPath, err.Error())) } } // Note: Bind-mount is not allowed for raw block device as there is no filesystem on it. - // So, we create softlink to the device file. TODO: mknode() instead ??? - // Ex: ln -s /dev/mpathbm - if err := os.Symlink(stagingDev.Device.AltFullPathName, targetPath); err != nil { - errMsg := fmt.Sprintf("Failed to create symlink %s to the device path %s, err: %v", + rawDeviceMinor, err := strconv.Atoi(stagingDev.Device.Minor) + if err != nil { + errMsg := fmt.Sprintf("Failed to retrieve device minor to create %s from the mpath device path %s, err: %v", + targetPath, stagingDev.Device.AltFullPathName, err.Error()) + log.Error("Error: ", errMsg) + return status.Error(codes.Internal, errMsg) + } + + rawDeviceMajor, err := strconv.Atoi(stagingDev.Device.Major) + if err != nil { + errMsg := fmt.Sprintf("Failed to retrieve device major to create %s from the mpath device path %s, err: %v", + targetPath, stagingDev.Device.AltFullPathName, err.Error()) + log.Error("Error: ", errMsg) + return status.Error(codes.Internal, errMsg) + } + + rawDeviceMode := uint32(unix.S_IFBLK | 0660) + rawDeviceMM := int(unix.Mkdev(uint32(rawDeviceMajor), uint32(rawDeviceMinor))) + + // Create block device + if err := unix.Mknod(targetPath, rawDeviceMode, rawDeviceMM); err != nil { + errMsg := fmt.Sprintf("Failed to create block device %s to the mpath device path %s, err: %v", targetPath, stagingDev.Device.AltFullPathName, err.Error()) log.Error("Error: ", errMsg) return status.Error(codes.Internal, errMsg) @@ -1456,10 +1475,10 @@ func (driver *Driver) isVolumePublished( return false, nil // Not published yet as targetPath does not exists } - // Check if target path is the symlink to the device - _, symlink, _ := util.IsFileSymlink(targetPath) - if !symlink { - log.Tracef("Target path %s is not symlink to the device %s", targetPath, stagingDev.Device.AltFullPathName) + // Check if target path is a block device + _, blockDevice, _ := util.IsFileBlockDevice(targetPath) + if !blockDevice { + log.Tracef("Target path %s is not a block device to the mpath device %s", targetPath, stagingDev.Device.AltFullPathName) return false, nil // Not published yet as symlink does not exists } @@ -1732,13 +1751,13 @@ func (driver *Driver) nodeUnpublishVolume(targetPath string) error { defer log.Trace("<<<<< nodeUnpublishVolume") // Block volume: Check for symlink and remove it - _, symlink, _ := util.IsFileSymlink(targetPath) - if symlink { - // Remove the symlink - log.Tracef("Removing the symlink from target path %s", targetPath) + _, blockDevice, _ := util.IsFileBlockDevice(targetPath) + if blockDevice { + // Remove the block device + log.Tracef("Removing the block device from mpath device %s", targetPath) if err := util.FileDelete(targetPath); err != nil { return status.Error(codes.Internal, - fmt.Sprintf("Error removing the symlink target path %s, err: %s", + fmt.Sprintf("Error removing the block device target path%s, err: %s", targetPath, err.Error())) } return nil From 0acbac0d7b115727d58235b534d0a292cea0c7f5 Mon Sep 17 00:00:00 2001 From: Michael Mattsson Date: Fri, 31 Oct 2025 09:57:48 -0700 Subject: [PATCH 2/2] Switch from symlink to block devices for raw block devices Signed-off-by: Michael Mattsson --- go.mod | 2 +- go.sum | 4 +-- .../common-host-libs/model/types.go | 34 ++++++++++++++----- .../csp/container_storage_provider.go | 21 ++++++++++++ .../fake/fake_storage_provider.go | 6 ++++ .../storageprovider/storage_provider.go | 6 ++-- .../hpe-storage/common-host-libs/util/file.go | 15 ++++++++ vendor/modules.txt | 2 +- 8 files changed, 75 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index 6b607952..2c60debd 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/Scalingo/go-etcd-lock/v5 v5.0.8 github.com/container-storage-interface/spec v1.7.0 github.com/golang/protobuf v1.5.4 - github.com/hpe-storage/common-host-libs v0.0.0-20250814050617-b92e778a7508 + github.com/hpe-storage/common-host-libs v0.0.0-20251030032113-e9caff71ad6b github.com/hpe-storage/k8s-custom-resources v0.0.0-20240118202512-5f62990a7c2d github.com/kubernetes-csi/csi-lib-utils v0.11.0 github.com/kubernetes-csi/csi-test v2.2.0+incompatible diff --git a/go.sum b/go.sum index 10f20fe5..2d58c339 100644 --- a/go.sum +++ b/go.sum @@ -273,8 +273,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFb github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/hpe-storage/common-host-libs v0.0.0-20250814050617-b92e778a7508 h1:urKXbbZW3Snld+VZqLgmnxjFH9cdiQAZPzpWz+sfMtw= -github.com/hpe-storage/common-host-libs v0.0.0-20250814050617-b92e778a7508/go.mod h1:LtFiruLqjTXLJ5TfFAYDEgSja5oW0rey5D1fSBTe2OY= +github.com/hpe-storage/common-host-libs v0.0.0-20251030032113-e9caff71ad6b h1:pfoON+u3bzyvxgVb43cTk2PvP6QXc7BZUw/cXh+pGiA= +github.com/hpe-storage/common-host-libs v0.0.0-20251030032113-e9caff71ad6b/go.mod h1:LtFiruLqjTXLJ5TfFAYDEgSja5oW0rey5D1fSBTe2OY= github.com/hpe-storage/k8s-custom-resources v0.0.0-20240118202512-5f62990a7c2d h1:ui+o/+dBn3pwOBHL3AB14XsYaeogpPJZF0Dk3vJh7zY= github.com/hpe-storage/k8s-custom-resources v0.0.0-20240118202512-5f62990a7c2d/go.mod h1:+zrGrGKf/jqr38KxLEGRll+UsjeUybdLH0MZCBxPPoI= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= diff --git a/vendor/github.com/hpe-storage/common-host-libs/model/types.go b/vendor/github.com/hpe-storage/common-host-libs/model/types.go index f049ccdd..b38c31bd 100644 --- a/vendor/github.com/hpe-storage/common-host-libs/model/types.go +++ b/vendor/github.com/hpe-storage/common-host-libs/model/types.go @@ -392,15 +392,15 @@ type Token struct { // Node represents a host that would access volumes through the CSP type Node struct { - ID string `json:"id,omitempty"` - Name string `json:"name,omitempty"` - UUID string `json:"uuid,omitempty"` - Iqns []*string `json:"iqns,omitempty"` - Networks []*string `json:"networks,omitempty"` - Wwpns []*string `json:"wwpns,omitempty"` - ChapUser string `json:"chap_user,omitempty"` - ChapPassword string `json:"chap_password,omitempty"` - AccessProtocol string `json:"access_protocol,omitempty"` + ID string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + UUID string `json:"uuid,omitempty"` + Iqns []*string `json:"iqns,omitempty"` + Networks []*string `json:"networks,omitempty"` + Wwpns []*string `json:"wwpns,omitempty"` + ChapUser string `json:"chap_user,omitempty"` + ChapPassword string `json:"chap_password,omitempty"` + AccessProtocol string `json:"access_protocol,omitempty"` } // KeyValue is a store of key-value pairs @@ -464,3 +464,19 @@ type ProcMount struct { FileSystem string `json:"FileSystem,omitempty"` Options string `json:"Options,omitempty"` } + +// PublishOptions are the options needed to publish a file volume +type PublishFileOptions struct { + Name string `json:"name,omitempty"` + HostUUID string `json:"host_uuid,omitempty"` + AccessProtocol string `json:"access_protocol,omitempty"` + VolumeID string `json:"volume_id,omitempty"` + AccessIP string `json:"access_ip,omitempty"` +} + +// PublishFileInfo is the node side data required to access a volume +type PublishFileInfo struct { + HostIP string `json:"host_ip,omitempty"` + MountPath string `json:"mount_path,omitempty"` + //TODO can contain more info if needed +} diff --git a/vendor/github.com/hpe-storage/common-host-libs/storageprovider/csp/container_storage_provider.go b/vendor/github.com/hpe-storage/common-host-libs/storageprovider/csp/container_storage_provider.go index 6d10a611..bb979d61 100644 --- a/vendor/github.com/hpe-storage/common-host-libs/storageprovider/csp/container_storage_provider.go +++ b/vendor/github.com/hpe-storage/common-host-libs/storageprovider/csp/container_storage_provider.go @@ -529,6 +529,27 @@ func (provider *ContainerStorageProvider) PublishVolume(id, hostUUID, accessProt return response, err } +// PublishFileVolume will make a volume visible (add an ACL) to the given host +func (provider *ContainerStorageProvider) PublishFileVolume(publishFileOptions *model.PublishFileOptions) (*model.PublishFileInfo, error) { + response := &model.PublishFileInfo{} + var errorResponse *ErrorsPayload + + status, err := provider.invoke( + &connectivity.Request{ + Action: "PUT", + Path: fmt.Sprintf("/containers/v1/volumes/%s/actions/publish", publishFileOptions.VolumeID), + Payload: publishFileOptions, + Response: &response, + ResponseError: &errorResponse, + }, + ) + if errorResponse != nil { + return nil, handleError(status, errorResponse) + } + + return response, err +} + // UnpublishVolume will make a volume invisible (remove an ACL) from the given host func (provider *ContainerStorageProvider) UnpublishVolume(id, hostUUID string) error { var errorResponse *ErrorsPayload diff --git a/vendor/github.com/hpe-storage/common-host-libs/storageprovider/fake/fake_storage_provider.go b/vendor/github.com/hpe-storage/common-host-libs/storageprovider/fake/fake_storage_provider.go index 9297a123..40ba13d8 100644 --- a/vendor/github.com/hpe-storage/common-host-libs/storageprovider/fake/fake_storage_provider.go +++ b/vendor/github.com/hpe-storage/common-host-libs/storageprovider/fake/fake_storage_provider.go @@ -287,3 +287,9 @@ func (provider *StorageProvider) EditVolume(id string, parameters map[string]int } return &fakeVolume, nil } + +func (provider *StorageProvider) PublishFileVolume(publishOptions *model.PublishFileOptions) (*model.PublishFileInfo, error) { + return &model.PublishFileInfo{ + HostIP: "eui.fake", + }, nil +} diff --git a/vendor/github.com/hpe-storage/common-host-libs/storageprovider/storage_provider.go b/vendor/github.com/hpe-storage/common-host-libs/storageprovider/storage_provider.go index 43ff00da..f3688843 100644 --- a/vendor/github.com/hpe-storage/common-host-libs/storageprovider/storage_provider.go +++ b/vendor/github.com/hpe-storage/common-host-libs/storageprovider/storage_provider.go @@ -4,9 +4,10 @@ package storageprovider import ( "fmt" + "strconv" + log "github.com/hpe-storage/common-host-libs/logger" "github.com/hpe-storage/common-host-libs/model" - "strconv" ) const ( @@ -35,7 +36,8 @@ type StorageProvider interface { CloneVolume(name, description, sourceID, snapshotID string, size int64, opts map[string]interface{}) (*model.Volume, error) DeleteVolume(id string, force bool) error PublishVolume(id, hostUUID, accessProtocol string) (*model.PublishInfo, error) // Idempotent - UnpublishVolume(id, hostUUID string) error // Idempotent + PublishFileVolume(publishOptions *model.PublishFileOptions) (*model.PublishFileInfo, error) + UnpublishVolume(id, hostUUID string) error // Idempotent ExpandVolume(id string, requestBytes int64) (*model.Volume, error) GetSnapshot(id string) (*model.Snapshot, error) GetSnapshotByName(name string, sourceVolID string) (*model.Snapshot, error) diff --git a/vendor/github.com/hpe-storage/common-host-libs/util/file.go b/vendor/github.com/hpe-storage/common-host-libs/util/file.go index 079cd851..b760ae0e 100644 --- a/vendor/github.com/hpe-storage/common-host-libs/util/file.go +++ b/vendor/github.com/hpe-storage/common-host-libs/util/file.go @@ -87,6 +87,21 @@ func IsFileSymlink(path string) (exists bool, symlink bool, err error) { return true, (info.Mode()&os.ModeSymlink == os.ModeSymlink), nil } +// IsFileBlockDevice to check if the path exists and is a raw device (block or symlink) +func IsFileBlockDevice(path string) (exists bool, device bool, err error) { + log.Tracef(">>>>> IsFileBlockDevice for path %s", path) + defer log.Trace("<<<<< IsFileBlockDevice") + + info, err := os.Lstat(path) + if err != nil { + if os.IsNotExist(err) { + return false, false, nil + } + return true, false, err + } + return true, (info.Mode()&os.ModeDevice == os.ModeDevice || info.Mode()&os.ModeSymlink == os.ModeSymlink), nil +} + // CreateDirIfNotExists to create a directory if it is not already available func CreateDirIfNotExists(dirPath string, perm os.FileMode) error { // Check if the directory already exists diff --git a/vendor/modules.txt b/vendor/modules.txt index 2e49f37e..feae7a14 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -89,7 +89,7 @@ github.com/google/uuid # github.com/gorilla/mux v1.8.0 ## explicit; go 1.12 github.com/gorilla/mux -# github.com/hpe-storage/common-host-libs v0.0.0-20250814050617-b92e778a7508 +# github.com/hpe-storage/common-host-libs v0.0.0-20251030032113-e9caff71ad6b ## explicit; go 1.23.0 github.com/hpe-storage/common-host-libs/chapi github.com/hpe-storage/common-host-libs/concurrent