feat: add resize fs (#382)

* feat: resize volume when staged

* chore: advertise VolumeExpansion as OFFLINE

* chore: move function to helper

* feat: resize fs on NodeExpandVolume

* test: ignore shallow err

* test: add mocking for Format and Resize

* test: improve NodeExpandVolume

* test(chainsaw): output CSI logs on error

* fix: nil checks

* fix: missing Formater

* chore: point to SafeFormatAndMount

* fix: test

* fix: resize deps

need resize2fs, dumpe2fs and xfs_io

* test: resizing disk size

* fix: validate input first

* test: expand read only

* test: df 9.8G

* test: fix pod create

* test: fix pod creation

* chore: revert grpc error

* test: partition

* chore: grpc error change

* test: getReadOnlyFromCapability vc nil

* chore: use grpc errors

* fix: return notfound on bad volumename

* chore: move functions in helpers/errors

* doc: offline resizing

Author: guilhem

.golangci.yml

@@ -156,6 +156,9 @@ issues:
       linters:
         - gosec
         - gas
+    
+    - text: 'shadow: declaration of "(err|ctx)" shadows declaration at'
+      linters: [govet]
 
   exclude-use-default: false
   new: false

Dockerfile

@@ -20,8 +20,8 @@ FROM alpine:3.20.3
 LABEL maintainers="Linode"
 LABEL description="Linode CSI Driver"
 
-RUN apk add --no-cache e2fsprogs findmnt blkid cryptsetup
-RUN apk add --no-cache xfsprogs=6.2.0-r2 --repository=http://dl-cdn.alpinelinux.org/alpine/v3.18/main
+RUN apk add --no-cache e2fsprogs e2fsprogs-extra findmnt blkid cryptsetup
+RUN apk add --no-cache xfsprogs=6.2.0-r2 xfsprogs-extra=6.2.0-r2 --repository=http://dl-cdn.alpinelinux.org/alpine/v3.18/main
 
 COPY --from=builder /bin/linode-blockstorage-csi-driver /linode
 

Dockerfile.dev

@@ -13,13 +13,15 @@ RUN apk add \
     cryptsetup-dev \
     curl \
     e2fsprogs \
+    e2fsprogs-extra \
     findmnt \
     gcc \
     lsblk \
     make \
     musl-dev \
     pkgconfig \
-    xfsprogs
+    xfsprogs \
+    xfsprogs-extra
 
 COPY go.mod go.sum ./
 RUN go mod tidy

docs/deployment.md

@@ -159,4 +159,10 @@ kubectl apply -f https://raw.githubusercontent.com/linode/linode-blockstorage-cs
 
    **Note:** To support this change, block storage volume attachments are no longer persisted across reboots.
 
-   <!-- Add note about volume resizing limitations -->
+4. **Offline Volume Resizing**
+
+    - The CSI driver supports offline volume resizing. This means that the volume must be unmounted from all nodes before resizing.
+    - To resize a volume, update the `spec.resources.requests.storage` field in the PersistentVolumeClaim (PVC) manifest and apply the changes.
+    - The CSI driver will automatically resize the underlying Linode Block Storage Volume to match the new size specified in the PVC.
+    - The volume must be unmounted from all nodes before resizing.  
+      It could be unmounted by deleting the pod using the volume or by using the `kubectl delete pod <pod-name>` command.

internal/driver/driver.go

@@ -25,11 +25,11 @@ import (
 	"github.com/container-storage-interface/spec/lib/go/csi"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
-	"k8s.io/mount-utils"
 
 	devicemanager "github.com/linode/linode-blockstorage-csi-driver/pkg/device-manager"
 	linodeclient "github.com/linode/linode-blockstorage-csi-driver/pkg/linode-client"
 	"github.com/linode/linode-blockstorage-csi-driver/pkg/logger"
+	mountmanager "github.com/linode/linode-blockstorage-csi-driver/pkg/mount-manager"
 	"github.com/linode/linode-blockstorage-csi-driver/pkg/observability"
 )
 
@@ -80,8 +80,9 @@ func GetLinodeDriver(ctx context.Context) *LinodeDriver {
 func (linodeDriver *LinodeDriver) SetupLinodeDriver(
 	ctx context.Context,
 	linodeClient linodeclient.LinodeClient,
-	mounter *mount.SafeFormatAndMount,
+	mounter *mountmanager.SafeFormatAndMount,
 	deviceUtils devicemanager.DeviceUtils,
+	resizeFs mountmanager.ResizeFSer,
 	metadata Metadata,
 	name,
 	vendorVersion,
@@ -118,7 +119,7 @@ func (linodeDriver *LinodeDriver) SetupLinodeDriver(
 	linodeDriver.volumeLabelPrefix = volumeLabelPrefix
 
 	log.V(2).Info("Setting up RPC Servers")
-	linodeDriver.ns, err = NewNodeServer(ctx, linodeDriver, mounter, deviceUtils, linodeClient, metadata, encrypt)
+	linodeDriver.ns, err = NewNodeServer(ctx, linodeDriver, mounter, deviceUtils, linodeClient, metadata, encrypt, resizeFs)
 	if err != nil {
 		return fmt.Errorf("new node server: %w", err)
 	}

internal/driver/driver_test.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/linode/linode-blockstorage-csi-driver/mocks"
 	linodeclient "github.com/linode/linode-blockstorage-csi-driver/pkg/linode-client"
+	mountmanager "github.com/linode/linode-blockstorage-csi-driver/pkg/mount-manager"
 )
 
 var (
@@ -30,14 +31,17 @@ func TestDriverSuite(t *testing.T) {
 	mockCtrl := gomock.NewController(t)
 	defer mockCtrl.Finish()
 
-	mounter := &mount.SafeFormatAndMount{
-		Interface: mocks.NewMockMounter(mockCtrl),
-		Exec:      mocks.NewMockExecutor(mockCtrl),
+	mounter := &mountmanager.SafeFormatAndMount{
+		SafeFormatAndMount: &mount.SafeFormatAndMount{
+			Interface: mocks.NewMockMounter(mockCtrl),
+			Exec:      mocks.NewMockExecutor(mockCtrl),
+		},
 	}
 	deviceUtils := mocks.NewMockDeviceUtils(mockCtrl)
 	fileSystem := mocks.NewMockFileSystem(mockCtrl)
 	cryptSetup := mocks.NewMockCryptSetupClient(mockCtrl)
 	encrypt := NewLuksEncryption(mounter.Exec, fileSystem, cryptSetup)
+	resizeFs := mocks.NewMockResizeFSer(mockCtrl)
 
 	fakeCloudProvider, err := linodeclient.NewLinodeClient("dummy", fmt.Sprintf("LinodeCSI/%s", vendorVersion), "")
 	if err != nil {
@@ -57,7 +61,7 @@ func TestDriverSuite(t *testing.T) {
 	metricsPort := "10251"
 	enableTracing := "true"
 	tracingPort := "4318"
-	if err := linodeDriver.SetupLinodeDriver(context.Background(), fakeCloudProvider, mounter, deviceUtils, md, driver, vendorVersion, bsPrefix, encrypt, enableMetrics, metricsPort, enableTracing, tracingPort); err != nil {
+	if err := linodeDriver.SetupLinodeDriver(context.Background(), fakeCloudProvider, mounter, deviceUtils, resizeFs, md, driver, vendorVersion, bsPrefix, encrypt, enableMetrics, metricsPort, enableTracing, tracingPort); err != nil {
 		t.Fatalf("Failed to setup Linode Driver: %v", err)
 	}
 

internal/driver/errors.go

@@ -1,6 +1,8 @@
 package driver
 
 import (
+	"errors"
+
 	"github.com/container-storage-interface/spec/lib/go/csi"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
@@ -53,6 +55,9 @@ var (
 	// operation was not specified, despite indicating a new volume should be
 	// created by cloning an existing one.
 	errNoSourceVolume = status.Error(codes.InvalidArgument, "no volume content source specified")
+
+	ErrNoVolumeCapability = errors.New("volume capability is required")
+	ErrNoAccessMode       = errors.New("access mode is nil")
 )
 
 // errRegionMismatch returns an error indicating a volume is in gotRegion, but

internal/driver/identityserver.go

@@ -105,7 +105,7 @@ func (linodeIdentity *IdentityServer) GetPluginCapabilities(ctx context.Context,
 					// 	2. Delete and recreate the pod that is using the PVC(or scale replicas accordingly)
 					// 	3. This operation should detach and re-attach the volume to the newly created pod allowing you to use the updated size
 					VolumeExpansion: &csi.PluginCapability_VolumeExpansion{
-						Type: csi.PluginCapability_VolumeExpansion_ONLINE,
+						Type: csi.PluginCapability_VolumeExpansion_OFFLINE,
 					},
 				},
 			},

internal/driver/identityserver_test.go

@@ -127,7 +127,7 @@ func TestIdentityServer_GetPluginCapabilities(t *testing.T) {
 		{
 			Type: &csi.PluginCapability_VolumeExpansion_{
 				VolumeExpansion: &csi.PluginCapability_VolumeExpansion{
-					Type: csi.PluginCapability_VolumeExpansion_ONLINE,
+					Type: csi.PluginCapability_VolumeExpansion_OFFLINE,
 				},
 			},
 		},

internal/driver/nodeserver.go

@@ -15,32 +15,35 @@ limitations under the License.
 */
 
 import (
-	"encoding/json"
+	"errors"
 	"fmt"
 	"strconv"
 	"sync"
 	"time"
 
 	"github.com/container-storage-interface/spec/lib/go/csi"
-	"github.com/linode/linodego"
 	"golang.org/x/net/context"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
 	"k8s.io/mount-utils"
 
 	devicemanager "github.com/linode/linode-blockstorage-csi-driver/pkg/device-manager"
 	filesystem "github.com/linode/linode-blockstorage-csi-driver/pkg/filesystem"
 	linodeclient "github.com/linode/linode-blockstorage-csi-driver/pkg/linode-client"
 	linodevolumes "github.com/linode/linode-blockstorage-csi-driver/pkg/linode-volumes"
 	"github.com/linode/linode-blockstorage-csi-driver/pkg/logger"
+	mountmanager "github.com/linode/linode-blockstorage-csi-driver/pkg/mount-manager"
 	"github.com/linode/linode-blockstorage-csi-driver/pkg/observability"
 )
 
 type NodeServer struct {
 	driver      *LinodeDriver
-	mounter     *mount.SafeFormatAndMount
+	mounter     *mountmanager.SafeFormatAndMount
 	deviceutils devicemanager.DeviceUtils
 	client      linodeclient.LinodeClient
 	metadata    Metadata
 	encrypt     Encryption
+	resizeFs    mountmanager.ResizeFSer
 	// TODO: Only lock mutually exclusive calls and make locking more fine grained
 	mux sync.Mutex
 
@@ -49,7 +52,7 @@ type NodeServer struct {
 
 var _ csi.NodeServer = &NodeServer{}
 
-func NewNodeServer(ctx context.Context, linodeDriver *LinodeDriver, mounter *mount.SafeFormatAndMount, deviceUtils devicemanager.DeviceUtils, client linodeclient.LinodeClient, metadata Metadata, encrypt Encryption) (*NodeServer, error) {
+func NewNodeServer(ctx context.Context, linodeDriver *LinodeDriver, mounter *mountmanager.SafeFormatAndMount, deviceUtils devicemanager.DeviceUtils, client linodeclient.LinodeClient, metadata Metadata, encrypt Encryption, resize mountmanager.ResizeFSer) (*NodeServer, error) {
 	log := logger.GetLogger(ctx)
 
 	log.V(4).Info("Creating new NodeServer")
@@ -78,6 +81,7 @@ func NewNodeServer(ctx context.Context, linodeDriver *LinodeDriver, mounter *mou
 		client:      client,
 		metadata:    metadata,
 		encrypt:     encrypt,
+		resizeFs:    resize,
 	}
 
 	log.V(4).Info("NodeServer created successfully")
@@ -199,25 +203,39 @@ func (ns *NodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol
 	ns.mux.Lock()
 	defer ns.mux.Unlock()
 
+	// Part 1: Validate request object
+
 	// Before to functionStartTime, validate the request object (NodeStageVolumeRequest)
 	log.V(4).Info("Validating request", "volumeID", volumeID)
 	if err := validateNodeStageVolumeRequest(ctx, req); err != nil {
 		observability.RecordMetrics(observability.NodeStageVolumeTotal, observability.NodeStageVolumeDuration, observability.Failed, functionStartTime)
-		return nil, err
+		return nil, status.Error(codes.InvalidArgument, err.Error())
+	}
+
+	// Part 2: Get information of attached device
+
+	readonly, err := getReadOnlyFromCapability(req.GetVolumeCapability())
+	if err != nil {
+		observability.RecordMetrics(observability.NodeStageVolumeTotal, observability.NodeStageVolumeDuration, observability.Failed, functionStartTime)
+		return nil, errors.Join(errInternal("failed to get readonly from volume capability: %v", err), err)
 	}
 
+	stagingTargetPath := req.GetStagingTargetPath()
+
 	// Get the LinodeVolumeKey which we need to find the device path
 	LinodeVolumeKey, err := linodevolumes.ParseLinodeVolumeKey(volumeID)
 	if err != nil {
 		observability.RecordMetrics(observability.NodeStageVolumeTotal, observability.NodeStageVolumeDuration, observability.Failed, functionStartTime)
-		return nil, err
+		return nil, errors.Join(status.Errorf(codes.InvalidArgument, "volume not found: %v", err), err)
 	}
 
 	// Get device path of attached device
 	partition := ""
 
-	if part, ok := req.GetVolumeContext()["partition"]; ok {
-		partition = part
+	if vc := req.GetVolumeContext(); vc != nil {
+		if part, ok := vc["partition"]; ok {
+			partition = part
+		}
 	}
 
 	log.V(4).Info("Finding device path", "volumeID", volumeID)
@@ -227,9 +245,10 @@ func (ns *NodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol
 		return nil, err
 	}
 
-	// Check if staging target path is a valid mount point.
-	log.V(4).Info("Ensuring staging target path is a valid mount point", "volumeID", volumeID, "stagingTargetPath", req.GetStagingTargetPath())
-	notMnt, err := ns.ensureMountPoint(ctx, req.GetStagingTargetPath(), filesystem.NewFileSystem())
+	// Part 3: check if staging target path is a valid mount point.
+
+	log.V(4).Info("Ensuring staging target path is a valid mount point", "volumeID", volumeID, "stagingTargetPath", stagingTargetPath)
+	notMnt, err := ns.ensureMountPoint(ctx, stagingTargetPath, filesystem.NewFileSystem())
 	if err != nil {
 		observability.RecordMetrics(observability.NodeStageVolumeTotal, observability.NodeStageVolumeDuration, observability.Failed, functionStartTime)
 		return nil, err
@@ -244,7 +263,7 @@ func (ns *NodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol
 
 		*/
 		observability.RecordMetrics(observability.NodeStageVolumeTotal, observability.NodeStageVolumeDuration, observability.Failed, functionStartTime)
-		log.V(4).Info("Staging target path is already a mount point", "volumeID", volumeID, "stagingTargetPath", req.GetStagingTargetPath())
+		log.V(4).Info("Staging target path is already a mount point", "volumeID", volumeID, "stagingTargetPath", stagingTargetPath)
 		return &csi.NodeStageVolumeResponse{}, nil
 	}
 
@@ -256,14 +275,27 @@ func (ns *NodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol
 		return &csi.NodeStageVolumeResponse{}, nil
 	}
 
-	// Mount device to stagingTargetPath
-	// If LUKS is enabled, format the device accordingly
-	log.V(4).Info("Mounting device", "volumeID", volumeID, "devicePath", devicePath, "stagingTargetPath", req.GetStagingTargetPath())
+	// Part 4: Mount device and format if needed
+
+	log.V(4).Info("Mounting device", "volumeID", volumeID, "devicePath", devicePath, "stagingTargetPath", stagingTargetPath)
 	if err := ns.mountVolume(ctx, devicePath, req); err != nil {
 		observability.RecordMetrics(observability.NodeStageVolumeTotal, observability.NodeStageVolumeDuration, observability.Failed, functionStartTime)
 		return nil, err
 	}
 
+	// Part 5: Resize fs
+
+	if !readonly {
+		resized, err := ns.resize(devicePath, stagingTargetPath)
+		if err != nil {
+			observability.RecordMetrics(observability.NodeStageVolumeTotal, observability.NodeStageVolumeDuration, observability.Failed, functionStartTime)
+			return nil, errInternal("failed to resize volume %s: %v", volumeID, err)
+		}
+		if resized {
+			log.V(4).Info("Successfully resized volume", "volumeID", volumeID)
+		}
+	}
+
 	// Record functionStatus metric
 	observability.RecordMetrics(observability.NodeStageVolumeTotal, observability.NodeStageVolumeDuration, observability.Completed, functionStartTime)
 
@@ -317,37 +349,72 @@ func (ns *NodeServer) NodeExpandVolume(ctx context.Context, req *csi.NodeExpandV
 	defer done()
 
 	functionStartTime := time.Now()
-	volumeID := req.GetVolumeId()
-	log.V(2).Info("Processing request", "volumeID", volumeID)
 
+	volumeID := req.GetVolumeId()
 	// Validate req (NodeExpandVolumeRequest)
+
 	log.V(4).Info("Validating request", "volumeID", volumeID)
 	if err := validateNodeExpandVolumeRequest(ctx, req); err != nil {
 		observability.RecordMetrics(observability.NodeExpandTotal, observability.NodeExpandDuration, observability.Failed, functionStartTime)
-		return nil, err
+		return nil, errors.Join(status.Error(codes.InvalidArgument, fmt.Sprintf("validation failed: %v", err)), err)
 	}
 
-	// Check linode to see if a give volume exists by volume ID
-	// Make call to linode api using the linode api client
+	log.V(2).Info("Processing request", "volumeID", volumeID)
+
 	LinodeVolumeKey, err := linodevolumes.ParseLinodeVolumeKey(volumeID)
 	log.V(4).Info("Processed LinodeVolumeKey", "LinodeVolumeKey", LinodeVolumeKey)
 	if err != nil {
-		// Node volume expansion is not supported yet. To meet the spec, we need to implement this.
-		// For now, we'll return a not found error.
-
 		observability.RecordMetrics(observability.NodeExpandTotal, observability.NodeExpandDuration, observability.Failed, functionStartTime)
-		return nil, errNotFound("volume not found: %v", err)
+		return nil, errors.Join(status.Errorf(codes.NotFound, "volume not found: %v", err), err)
+	}
+
+	// We have no context for the partition, so we'll leave it empty
+	partition := ""
+
+	volumePath := req.GetVolumePath()
+	if volumePath == "" {
+		observability.RecordMetrics(observability.NodeExpandTotal, observability.NodeExpandDuration, observability.Completed, functionStartTime)
+		return nil, status.Error(codes.InvalidArgument, "volume path must be provided")
+	}
+
+	volumeCapability := req.GetVolumeCapability()
+	// VolumeCapability is optional, if specified, use that as source of truth
+	if volumeCapability != nil {
+		if blk := volumeCapability.GetBlock(); blk != nil {
+			// Noop for Block NodeExpandVolume
+			log.V(4).Info("NodeExpandVolume: called. Since it is a block device, ignoring...", "volumeID", volumeID, "volumePath", volumePath)
+			observability.RecordMetrics(observability.NodeExpandTotal, observability.NodeExpandDuration, observability.Completed, functionStartTime)
+			return &csi.NodeExpandVolumeResponse{}, nil
+		}
+
+		readonly, err := getReadOnlyFromCapability(volumeCapability)
+		if err != nil {
+			observability.RecordMetrics(observability.NodeExpandTotal, observability.NodeExpandDuration, observability.Failed, functionStartTime)
+			return nil, errInternal("failed to check if capability for volume %s is readonly: %v", volumeID, err)
+		}
+		if readonly {
+			log.V(4).Info("NodeExpandVolume succeeded", "volumeID", volumeID, "volumePath", volumePath)
+			observability.RecordMetrics(observability.NodeExpandTotal, observability.NodeExpandDuration, observability.Completed, functionStartTime)
+			return &csi.NodeExpandVolumeResponse{}, nil
+		}
 	}
-	jsonFilter, err := json.Marshal(map[string]string{"label": LinodeVolumeKey.Label})
+
+	devicePath, err := ns.findDevicePath(ctx, *LinodeVolumeKey, partition)
 	if err != nil {
 		observability.RecordMetrics(observability.NodeExpandTotal, observability.NodeExpandDuration, observability.Failed, functionStartTime)
-		return nil, errInternal("marshal json filter: %v", err)
+		return nil, err
 	}
 
-	log.V(4).Info("Listing volumes", "volumeID", volumeID)
-	if _, err = ns.client.ListVolumes(ctx, linodego.NewListOptions(0, string(jsonFilter))); err != nil {
+	// Resize the volume
+
+	resized, err := ns.resize(devicePath, volumePath)
+	if err != nil {
 		observability.RecordMetrics(observability.NodeExpandTotal, observability.NodeExpandDuration, observability.Failed, functionStartTime)
-		return nil, errVolumeNotFound(LinodeVolumeKey.VolumeID)
+		return nil, errInternal("failed to resize volume %s: %v", volumeID, err)
+	}
+
+	if resized {
+		log.V(4).Info("Successfully resized volume", "volumeID", volumeID)
 	}
 
 	// Record functionStatus metric