NVMe LUKS Support

Author: jwebster7

frontend/csi/controller_server.go

@@ -1,4 +1,4 @@
-// Copyright 2022 NetApp, Inc. All Rights Reserved.
+// Copyright 2024 NetApp, Inc. All Rights Reserved.
 
 package csi
 

frontend/csi/node_server.go

@@ -1536,7 +1536,7 @@ func (p *Plugin) nodeStageISCSIVolume(
 		}
 
 		var luksDevice models.LUKSDeviceInterface
-		luksDevice, err = utils.NewLUKSDevice(publishInfo.DevicePath, req.VolumeContext["internalName"])
+		luksDevice, err = p.devices.NewLUKSDevice(publishInfo.DevicePath, req.VolumeContext["internalName"])
 		if err != nil {
 			return err
 		}
@@ -1836,11 +1836,13 @@ func (p *Plugin) nodePublishISCSIVolume(
 			luksDevice, err = p.devices.NewLUKSDeviceFromMappingPath(ctx, devicePath,
 				req.VolumeContext["internalName"])
 		} else {
-			luksDevice, err = utils.NewLUKSDevice(publishInfo.DevicePath, req.VolumeContext["internalName"])
+			luksDevice, err = p.devices.NewLUKSDevice(publishInfo.DevicePath, req.VolumeContext["internalName"])
 		}
+
 		if err != nil {
 			return nil, status.Error(codes.Internal, err.Error())
 		}
+
 		err = ensureLUKSVolumePassphrase(ctx, p.restClient, luksDevice, req.GetVolumeId(), req.GetSecrets(), false)
 		if err != nil {
 			Logc(ctx).WithError(err).Error("Failed to ensure current LUKS passphrase.")
@@ -2441,16 +2443,18 @@ func (p *Plugin) nodeStageNVMeVolume(
 	ctx context.Context, req *csi.NodeStageVolumeRequest,
 	publishInfo *models.VolumePublishInfo,
 ) error {
-	isLUKS := utils.ParseBool(publishInfo.LUKSEncryption)
+	isLUKS := utils.ParseBool(req.PublishContext["LUKSEncryption"])
 	publishInfo.LUKSEncryption = strconv.FormatBool(isLUKS)
 	publishInfo.MountOptions = req.PublishContext["mountOptions"]
 	publishInfo.NVMeSubsystemNQN = req.PublishContext["nvmeSubsystemNqn"]
 	publishInfo.NVMeNamespaceUUID = req.PublishContext["nvmeNamespaceUUID"]
 	publishInfo.NVMeTargetIPs = strings.Split(req.PublishContext["nvmeTargetIPs"], ",")
 	publishInfo.SANType = req.PublishContext["SANType"]
 
-	if err := utils.AttachNVMeVolumeRetry(ctx, req.VolumeContext["internalName"], "", publishInfo, nil,
-		utils.NVMeAttachTimeout); err != nil {
+	err := utils.AttachNVMeVolumeRetry(
+		ctx, req.VolumeContext["internalName"], "", publishInfo, req.GetSecrets(), utils.NVMeAttachTimeout,
+	)
+	if err != nil {
 		return err
 	}
 
@@ -2460,11 +2464,11 @@ func (p *Plugin) nodeStageNVMeVolume(
 	}
 
 	if isLUKS {
-		luksDevice, err := p.devices.NewLUKSDeviceFromMappingPath(ctx, publishInfo.DevicePath,
-			req.VolumeContext["internalName"])
+		luksDevice, err := p.devices.NewLUKSDevice(publishInfo.DevicePath, req.VolumeContext["internalName"])
 		if err != nil {
 			return err
 		}
+
 		// Ensure we update the passphrase in case it has never been set before
 		err = ensureLUKSVolumePassphrase(ctx, p.restClient, luksDevice, volumeId, req.GetSecrets(), true)
 		if err != nil {
@@ -2500,12 +2504,42 @@ func (p *Plugin) nodeUnstageNVMeVolume(
 		publishInfo.NVMeNamespaceUUID)
 
 	// Get the device using 'nvme-cli' commands. Flush the device IOs.
+	// Proceed further with unstage flow, if device is not found.
 	nvmeDev, err := p.nvmeHandler.NewNVMeDevice(ctx, publishInfo.NVMeNamespaceUUID)
-	// Proceed further with Unstage flow, if 'device is not found'.
 	if err != nil && !errors.IsNotFoundError(err) {
 		return nil, fmt.Errorf("error while getting NVMe device, %v", err)
 	}
 
+	var devicePath string
+	if nvmeDev != nil {
+		devicePath = nvmeDev.GetPath()
+	}
+
+	var luksMapperPath string
+	if utils.ParseBool(publishInfo.LUKSEncryption) && devicePath != "" {
+		fields := LogFields{
+			"lunID":           publishInfo.IscsiLunNumber,
+			"publishedDevice": publishInfo.DevicePath,
+			"nvmeDevPath":     nvmeDev.GetPath(),
+		}
+
+		luksMapperPath, err = p.devices.GetLUKSDeviceForMultipathDevice(devicePath)
+		if err != nil {
+			return &csi.NodeUnstageVolumeResponse{}, err
+		}
+
+		// Ensure the LUKS device is closed if the luksMapperPath is set.
+		if luksMapperPath != "" {
+			if err = p.devices.EnsureLUKSDeviceClosedWithMaxWaitLimit(ctx, luksMapperPath); err != nil {
+				if !errors.IsMaxWaitExceededError(err) {
+					Logc(ctx).WithFields(fields).WithError(err).Error("Failed to close LUKS device.")
+					return &csi.NodeUnstageVolumeResponse{}, err
+				}
+				Logc(ctx).WithFields(fields).WithError(err).Debug("LUKS close wait time exceeded, continuing with device removal.")
+			}
+		}
+	}
+
 	if !nvmeDev.IsNil() {
 		// If device is found, proceed to flush and clean up.
 		err := nvmeDev.FlushDevice(ctx, p.unsafeDetach, force)
@@ -2580,6 +2614,20 @@ func (p *Plugin) nodeUnstageNVMeVolume(
 		return nil, status.Error(codes.Internal, errStr)
 	}
 
+	// If the luks device still exists, it means the device was unable to be closed prior to removing the block
+	// device. This can happen if the LUN was deleted or offline. It should be removable by this point.
+	// It needs to be removed prior to removing the 'unmappedMpathDevice' device below.
+	if luksMapperPath != "" {
+		// EnsureLUKSDeviceClosed will not return an error if the device is already closed or removed.
+		if err = p.devices.EnsureLUKSDeviceClosed(ctx, luksMapperPath); err != nil {
+			Logc(ctx).WithFields(LogFields{
+				"devicePath": luksMapperPath,
+			}).WithError(err).Warning("Unable to remove LUKS mapper device.")
+		}
+		// Clear the time duration for the LUKS device.
+		utils.LuksCloseDurations.RemoveDurationTracking(luksMapperPath)
+	}
+
 	// Delete the device info we saved to the volume tracking info path so unstage can succeed.
 	if err := p.nodeHelper.DeleteTrackingInfo(ctx, volumeId); err != nil {
 		return nil, status.Error(codes.Internal, err.Error())
@@ -2608,36 +2656,39 @@ func (p *Plugin) nodePublishNVMeVolume(
 		publishInfo.MountOptions = utils.AppendToStringList(publishInfo.MountOptions, "ro", ",")
 	}
 
+	devicePath := publishInfo.DevicePath
 	if utils.ParseBool(publishInfo.LUKSEncryption) {
 		// Rotate the LUKS passphrase if needed, on failure, log and continue to publish
-		luksDevice, err := p.devices.NewLUKSDeviceFromMappingPath(ctx, publishInfo.DevicePath,
-			req.VolumeContext["internalName"])
+		luksDevice, err := p.devices.NewLUKSDevice(devicePath, req.VolumeContext["internalName"])
 		if err != nil {
 			return nil, status.Error(codes.Internal, err.Error())
 		}
+
 		err = ensureLUKSVolumePassphrase(ctx, p.restClient, luksDevice, req.GetVolumeId(), req.GetSecrets(), false)
 		if err != nil {
 			Logc(ctx).WithError(err).Error("Failed to ensure current LUKS passphrase.")
 		}
+
+		// At this point, we must reassign the device path to the luks mapper path for mounts to work.
+		devicePath = luksDevice.MappedDevicePath()
 	}
 
 	isRawBlock := publishInfo.FilesystemType == filesystem.Raw
 	if isRawBlock {
-
 		if len(publishInfo.MountOptions) > 0 {
 			publishInfo.MountOptions = utils.AppendToStringList(publishInfo.MountOptions, "bind", ",")
 		} else {
 			publishInfo.MountOptions = "bind"
 		}
 
 		// Place the block device at the target path for the raw-block.
-		err = p.mount.MountDevice(ctx, publishInfo.DevicePath, req.TargetPath, publishInfo.MountOptions, true)
+		err = p.mount.MountDevice(ctx, devicePath, req.TargetPath, publishInfo.MountOptions, true)
 		if err != nil {
 			return nil, status.Errorf(codes.Internal, "unable to bind mount raw device; %s", err)
 		}
 	} else {
 		// Mount the device.
-		err = p.mount.MountDevice(ctx, publishInfo.DevicePath, req.TargetPath, publishInfo.MountOptions, false)
+		err = p.mount.MountDevice(ctx, devicePath, req.TargetPath, publishInfo.MountOptions, false)
 		if err != nil {
 			return nil, status.Errorf(codes.Internal, "unable to mount device; %s", err)
 		}

storage_drivers/ontap/api/abstraction_rest.go

@@ -1,4 +1,4 @@
-// Copyright 2023 NetApp, Inc. All Rights Reserved.
+// Copyright 2024 NetApp, Inc. All Rights Reserved.
 
 package api
 
@@ -3231,11 +3231,11 @@ func (d OntapAPIREST) NVMeSubsystemCreate(ctx context.Context, subsystemName str
 	fields := []string{"target_nqn"}
 	subsystem, err := d.api.NVMeSubsystemGetByName(ctx, subsystemName, fields)
 	if err != nil {
-		Logc(ctx).Infof("problem getting subsystem; %v", err)
+		Logc(ctx).Infof("Problem getting subsystem; %v", err)
 		return nil, err
 	}
 	if subsystem == nil {
-		Logc(ctx).Infof("subsystem doesn't exists, creating new subsystem %v now.", subsystemName)
+		Logc(ctx).Infof("Subsystem doesn't exist, creating new subsystem %v now.", subsystemName)
 		subsystem, err = d.api.NVMeSubsystemCreate(ctx, subsystemName)
 		if err != nil {
 			return nil, err

storage_drivers/ontap/ontap_san.go

@@ -357,6 +357,7 @@ func (d *SANStorageDriver) Create(
 	if err != nil {
 		return fmt.Errorf("could not convert volume size %s: %v", volConfig.Size, err)
 	}
+
 	requestedSizeBytes, err := strconv.ParseUint(requestedSize, 10, 64)
 	if err != nil {
 		return fmt.Errorf("%v is an invalid volume size: %v", volConfig.Size, err)
@@ -367,8 +368,8 @@ func (d *SANStorageDriver) Create(
 	// part of the LUN but is not reported to the orchestrator.
 	reportedSize := lunSizeBytes
 	lunSizeBytes = incrementWithLUKSMetadataIfLUKSEnabled(ctx, lunSizeBytes, luksEncryption)
-
 	lunSize := strconv.FormatUint(lunSizeBytes, 10)
+
 	// Get the flexvol size based on the snapshot reserve
 	flexvolSize := drivers.CalculateVolumeSizeBytes(ctx, name, lunSizeBytes, snapshotReserveInt)
 	// Add extra 10% to the Flexvol to account for LUN metadata

storage_drivers/ontap/ontap_san_nvme.go

@@ -1,4 +1,4 @@
-// Copyright 2023 NetApp, Inc. All Rights Reserved.
+// Copyright 2024 NetApp, Inc. All Rights Reserved.
 
 package ontap
 
@@ -298,9 +298,19 @@ func (d *NVMeStorageDriver) Create(
 	if err != nil {
 		return fmt.Errorf("could not convert volume size %s: %v", volConfig.Size, err)
 	}
-	requestedSizeBytes, _ := strconv.ParseUint(requestedSize, 10, 64)
+
+	requestedSizeBytes, err := strconv.ParseUint(requestedSize, 10, 64)
+	if err != nil {
+		return fmt.Errorf("%v is an invalid volume size: %v", volConfig.Size, err)
+	}
 	namespaceSizeBytes := GetVolumeSize(requestedSizeBytes, storagePool.InternalAttributes()[Size])
+
+	// Add a constant overhead for LUKS volumes to account for LUKS metadata. This overhead is
+	// part of the LUN but is not reported to the orchestrator.
+	reportedSize := namespaceSizeBytes
+	namespaceSizeBytes = incrementWithLUKSMetadataIfLUKSEnabled(ctx, namespaceSizeBytes, luksEncryption)
 	namespaceSize := strconv.FormatUint(namespaceSizeBytes, 10)
+
 	// Get the FlexVol size based on the snapshot reserve.
 	flexVolSize := drivers.CalculateVolumeSizeBytes(ctx, name, namespaceSizeBytes, snapshotReserveInt)
 	// Add extra 10% to the FlexVol to account for Namespace metadata.
@@ -335,7 +345,7 @@ func (d *NVMeStorageDriver) Create(
 	}
 
 	// Update config to reflect values used to create volume.
-	volConfig.Size = strconv.FormatUint(namespaceSizeBytes, 10)
+	volConfig.Size = strconv.FormatUint(reportedSize, 10)
 	volConfig.SpaceReserve = spaceReserve
 	volConfig.SnapshotPolicy = snapshotPolicy
 	volConfig.SnapshotReserve = snapshotReserve
@@ -617,7 +627,9 @@ func (d *NVMeStorageDriver) Import(ctx context.Context, volConfig *storage.Volum
 	}
 
 	// Set the volume to LUKS if backend has LUKS true as default
-	volConfig.LUKSEncryption = d.Config.LUKSEncryption
+	if volConfig.LUKSEncryption == "" {
+		volConfig.LUKSEncryption = d.Config.LUKSEncryption
+	}
 
 	// Set the filesystem type to backend's, if it hasn't been set via annotation
 	// in the provided pvc during import.
@@ -649,6 +661,16 @@ func (d *NVMeStorageDriver) Import(ctx context.Context, volConfig *storage.Volum
 	// Use the Namespace size
 	volConfig.Size = nsInfo.Size
 
+	// If the import is a LUKS encrypted volume, then remove the LUKS metadata overhead from the reported
+	// size on the volConfig.
+	if utils.ParseBool(volConfig.LUKSEncryption) {
+		newSize, err := subtractUintFromSizeString(volConfig.Size, utils.LUKSMetadataSize)
+		if err != nil {
+			return err
+		}
+		volConfig.Size = newSize
+	}
+
 	// Rename the volume if Trident will manage its lifecycle
 	if !volConfig.ImportNotManaged {
 		err = d.API.VolumeRename(ctx, originalName, volConfig.InternalName)
@@ -1314,6 +1336,9 @@ func (d *NVMeStorageDriver) Resize(
 		return fmt.Errorf("requested size %d is less than existing volume size %d", requestedSizeBytes, nsSizeBytes)
 	}
 
+	// Add a constant overhead for LUKS volumes to account for LUKS metadata.
+	requestedSizeBytes = incrementWithLUKSMetadataIfLUKSEnabled(ctx, requestedSizeBytes, volConfig.LUKSEncryption)
+
 	snapshotReserveInt, err := getSnapshotReserveFromOntap(ctx, name, d.API.VolumeInfo)
 	if err != nil {
 		Logc(ctx).WithField("name", name).Errorf("Could not get the snapshot reserve percentage for volume.")
@@ -1376,6 +1401,9 @@ func (d *NVMeStorageDriver) Resize(
 		}
 	}
 
+	// LUKS metadata size is not reported so remove it from LUN size
+	requestedSizeBytes = decrementWithLUKSMetadataIfLUKSEnabled(ctx, requestedSizeBytes, volConfig.LUKSEncryption)
+
 	// Setting the new size in the volume config.
 	volConfig.Size = strconv.FormatUint(requestedSizeBytes, 10)
 	return nil

storage_drivers/ontap/ontap_san_nvme_test.go

@@ -1,11 +1,12 @@
-// Copyright 2023 NetApp, Inc. All Rights Reserved.
+// Copyright 2024 NetApp, Inc. All Rights Reserved.
 
 package ontap
 
 import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"strconv"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -18,6 +19,7 @@ import (
 	drivers "github.com/netapp/trident/storage_drivers"
 	"github.com/netapp/trident/storage_drivers/ontap/api"
 	"github.com/netapp/trident/storage_drivers/ontap/awsapi"
+	"github.com/netapp/trident/utils"
 	"github.com/netapp/trident/utils/errors"
 	"github.com/netapp/trident/utils/filesystem"
 	"github.com/netapp/trident/utils/models"
@@ -980,6 +982,21 @@ func TestNVMeCreate_NamespaceCreateAPIError(t *testing.T) {
 	assert.ErrorContains(t, err, "failed to create namespace")
 }
 
+func TestNVMeCreate_LUKSVolume(t *testing.T) {
+	d, mAPI := newNVMeDriverAndMockApi(t)
+	pool1, volConfig, volAttrs := getNVMeCreateArgs(d)
+
+	volConfig.LUKSEncryption = "true"
+	mAPI.EXPECT().VolumeExists(ctx, volConfig.InternalName).Return(false, nil)
+	mAPI.EXPECT().TieringPolicyValue(ctx).Return("TPolicy")
+	mAPI.EXPECT().VolumeCreate(ctx, gomock.Any()).Return(nil)
+	mAPI.EXPECT().NVMeNamespaceCreate(ctx, gomock.Any()).Return("nsUUID", nil)
+
+	err := d.Create(ctx, volConfig, pool1, volAttrs)
+
+	assert.NoError(t, err, "Failed to create NVMe volume.")
+}
+
 func TestNVMeCreate_Success(t *testing.T) {
 	d, mAPI := newNVMeDriverAndMockApi(t)
 	pool1, volConfig, volAttrs := getNVMeCreateArgs(d)
@@ -2248,6 +2265,52 @@ func TestImport(t *testing.T) {
 	assert.NoError(t, err)
 }
 
+func TestImport_LUKSNamespace(t *testing.T) {
+	d, mAPI := newNVMeDriverAndMockApi(t)
+	_, volConfig, _ := getNVMeCreateArgs(d)
+	originalName := "fakeOriginalName"
+	vol := &api.Volume{Aggregates: []string{"data"}}
+	ns := &api.NVMeNamespace{
+		Name: "/vol/cloneVol1/namespace0",
+		Size: "20GB",
+		UUID: "fakeUUID",
+	}
+	ns.State = "online"
+
+	vol.Comment = "fakeComment"
+	volConfig.LUKSEncryption = "true"
+	volConfig.ImportNotManaged = true
+	volConfig.Size = "20GB"
+	mAPI.EXPECT().VolumeInfo(ctx, gomock.Any()).Return(vol, nil)
+	mAPI.EXPECT().NVMeNamespaceGetByName(ctx, "/vol/"+originalName+"/*").Return(ns, nil)
+	mAPI.EXPECT().NVMeIsNamespaceMapped(ctx, "", ns.UUID).Return(false, nil)
+	// mAPI.EXPECT().VolumeRename(ctx, originalName, volConfig.InternalName).Return(nil)
+
+	beforeLUKSOverheadBytesStr, err := utils.ConvertSizeToBytes(volConfig.Size)
+	if err != nil {
+		t.Fatalf("failed to convert volume size")
+	}
+	beforeLUKSOverhead, err := strconv.ParseUint(beforeLUKSOverheadBytesStr, 10, 64)
+	if err != nil {
+		t.Fatalf("failed to convert volume size")
+	}
+
+	err = d.Import(ctx, volConfig, originalName)
+
+	afterLUKSOverheadBytesStr, err := utils.ConvertSizeToBytes(volConfig.Size)
+	if err != nil {
+		t.Fatalf("failed to convert volume size")
+	}
+	afterLUKSOverhead, err := strconv.ParseUint(afterLUKSOverheadBytesStr, 10, 64)
+	if err != nil {
+		t.Fatalf("failed to convert volume size")
+	}
+
+	assert.NoError(t, err)
+	assert.Less(t, afterLUKSOverhead, beforeLUKSOverhead)
+	assert.Equal(t, beforeLUKSOverhead, incrementWithLUKSMetadataIfLUKSEnabled(ctx, afterLUKSOverhead, "true"))
+}
+
 func TestImport_NameTemplate(t *testing.T) {
 	d, mAPI := newNVMeDriverAndMockApi(t)
 	_, volConfig, _ := getNVMeCreateArgs(d)