add nvme fault points

Add NVMe fault points for fault injection testing

Author: jharrod

frontend/csi/node_server.go

@@ -2866,6 +2866,10 @@ func (p *Plugin) nodeStageNVMeVolume(
 	}
 
 	if isLUKS {
+		if err := betweenAttachAndLUKSPassphrase.Inject(); err != nil {
+			return err
+		}
+
 		luksDevice := luks.NewDevice(publishInfo.DevicePath, req.VolumeContext["internalName"], p.command)
 
 		// Ensure we update the passphrase in case it has never been set before
@@ -2886,11 +2890,15 @@ func (p *Plugin) nodeStageNVMeVolume(
 		return err
 	}
 
+	if err := afterInitialTrackingInfoWrite.Inject(); err != nil {
+		return err
+	}
+
 	p.nvmeHandler.AddPublishedNVMeSession(&publishedNVMeSessions, publishInfo)
 	return nil
 }
 
-// nodeUnstageNVMEVolume unstages volume for nvme driver.
+// nodeUnstageNVMeVolume unstages volume for nvme driver.
 // - Get the device from tracking info OR from "nvme netapp ontapdevices".
 // - Flush the device using "nvme flush /dev/<nvme-device>"
 // - If the subsystem has <= 1 Namespaces connected to it, then disconnect.

utils/nvme/nvme.go

@@ -13,6 +13,7 @@ import (
 	"github.com/spf13/afero"
 	"go.uber.org/multierr"
 
+	"github.com/netapp/trident/internal/fiji"
 	. "github.com/netapp/trident/logging"
 	"github.com/netapp/trident/pkg/convert"
 	"github.com/netapp/trident/utils/devices"
@@ -25,6 +26,8 @@ import (
 
 const NVMeAttachTimeout = 20 * time.Second
 
+var beforeNVMeFlushDevice = fiji.Register("beforeNVMeFlushDevice", "nvme")
+
 func NewNVMeSubsystem(nqn string, command exec.Command, fs afero.Fs) *NVMeSubsystem {
 	return NewNVMeSubsystemDetailed(nqn, "", []Path{}, command, fs)
 }
@@ -177,6 +180,10 @@ func (d *NVMeDevice) GetPath() string {
 
 // FlushDevice flushes any ongoing IOs on the device.
 func (d *NVMeDevice) FlushDevice(ctx context.Context, ignoreErrors, force bool) error {
+	if err := beforeNVMeFlushDevice.Inject(); err != nil {
+		return err
+	}
+
 	// Force is set in forced detach use case. We don't flush in that scenario and return success.
 	if force {
 		return nil

utils/nvme/nvme_linux.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/spf13/afero"
 
+	"github.com/netapp/trident/internal/fiji"
 	. "github.com/netapp/trident/logging"
 	sa "github.com/netapp/trident/storage_attribute"
 	"github.com/netapp/trident/utils/errors"
@@ -23,6 +24,8 @@ var (
 	transport    = "tcp"
 	nvmeNQNRegex = regexp.MustCompile(`^nvme([0-9]+)n([0-9]+)$`)
 	nvmeRegex    = regexp.MustCompile(`^nvme([0-9]+)$`)
+
+	beforeNVMeDisconnect = fiji.Register("beforeNVMeDisconnect", "nvme_linux")
 )
 
 const (
@@ -162,6 +165,10 @@ func (s *NVMeSubsystem) DisconnectSubsystemFromHost(ctx context.Context) error {
 	Logc(ctx).Debug(">>>> nvme_linux.DisconnectSubsystemFromHost")
 	defer Logc(ctx).Debug("<<<< nvme_linux.DisconnectSubsystemFromHost")
 
+	if err := beforeNVMeDisconnect.Inject(); err != nil {
+		return err
+	}
+
 	_, err := s.command.Execute(ctx, "nvme", "disconnect", "-n", s.NQN)
 	if err != nil {
 		Logc(ctx).WithError(err).Errorf("Failed to disconnect subsystem %s.", s.NQN)