diff --git a/pkg/driver/controller_server.go b/pkg/driver/controller_server.go index d94ffb59..a4750017 100644 --- a/pkg/driver/controller_server.go +++ b/pkg/driver/controller_server.go @@ -924,7 +924,9 @@ func (driver *Driver) controllerPublishVolume( requestedAccessProtocol = iscsi } else if requestedAccessProtocol == "fc" { requestedAccessProtocol = fc - } + } else if requestedAccessProtocol == "nvmetcp" { + requestedAccessProtocol = "nvmetcp" + } if existingNode != nil { log.Tracef("CSP has already been notified about the node with ID %s and UUID %s", existingNode.ID, existingNode.UUID) @@ -957,12 +959,33 @@ func (driver *Driver) controllerPublishVolume( // TODO: add any additional info necessary to mount the device publishContext := map[string]string{} - publishContext[serialNumberKey] = publishInfo.SerialNumber - publishContext[accessProtocolKey] = publishInfo.AccessInfo.BlockDeviceAccessInfo.AccessProtocol - publishContext[targetNamesKey] = strings.Join(publishInfo.AccessInfo.BlockDeviceAccessInfo.TargetNames, ",") - publishContext[targetScopeKey] = requestedTargetScope - publishContext[lunIDKey] = strconv.Itoa(int(publishInfo.AccessInfo.BlockDeviceAccessInfo.LunID)) + + // NVMe over TCP support + if strings.EqualFold(publishInfo.AccessInfo.BlockDeviceAccessInfo.AccessProtocol, "nvmetcp") { + publishContext[serialNumberKey] = publishInfo.SerialNumber + publishContext[accessProtocolKey] = "nvmetcp" + // NQN, target address, and port for NVMe/TCP + if len(publishInfo.AccessInfo.BlockDeviceAccessInfo.TargetNames) > 0 { + publishContext[targetNamesKey] = publishInfo.AccessInfo.BlockDeviceAccessInfo.TargetNames[0] + } + if len(publishInfo.AccessInfo.BlockDeviceAccessInfo.NvmetcpAccessInfo.DiscoveryIPs) > 0 { + publishContext[discoveryIPsKey] = publishInfo.AccessInfo.BlockDeviceAccessInfo.NvmetcpAccessInfo.DiscoveryIPs[0] + } + if publishInfo.AccessInfo.BlockDeviceAccessInfo.NvmetcpAccessInfo.TargetPort != "" { + publishContext[targetPortKey] = publishInfo.AccessInfo.BlockDeviceAccessInfo.NvmetcpAccessInfo.TargetPort + } else { + publishContext[targetPortKey] = "4420" // default NVMe/TCP port + } + + }else{ + publishContext[serialNumberKey] = publishInfo.SerialNumber + publishContext[accessProtocolKey] = publishInfo.AccessInfo.BlockDeviceAccessInfo.AccessProtocol + publishContext[targetNamesKey] = strings.Join(publishInfo.AccessInfo.BlockDeviceAccessInfo.TargetNames, ",") + publishContext[targetScopeKey] = requestedTargetScope + publishContext[lunIDKey] = strconv.Itoa(int(publishInfo.AccessInfo.BlockDeviceAccessInfo.LunID)) + } + // Start of population of target array details if publishInfo.AccessInfo.BlockDeviceAccessInfo.SecondaryBackendDetails.PeerArrayDetails != nil { secondaryArrayMarshalledStr, err := json.Marshal(&publishInfo.AccessInfo.BlockDeviceAccessInfo.SecondaryBackendDetails) diff --git a/pkg/driver/node_server.go b/pkg/driver/node_server.go index 0f69493d..0d1a1de1 100644 --- a/pkg/driver/node_server.go +++ b/pkg/driver/node_server.go @@ -41,6 +41,8 @@ var ( const ( fileHostIPKey = "hostIP" + nvmetcp = "nvmetcp" + targetPortKey = "targetPort" ) var isWatcherEnabled = false @@ -572,6 +574,41 @@ func (driver *Driver) setupDevice( log.Tracef(">>>>> setupDevice, volumeID: %s, publishContext: %v", volumeID, log.MapScrubber(publishContext)) defer log.Trace("<<<<< setupDevice") + // Handle NVMe over TCP volumes + if publishContext[accessProtocolKey] == "nvmetcp" { + volume := &model.Volume{ + SerialNumber: publishContext[serialNumberKey], + AccessProtocol: publishContext[accessProtocolKey], + Nqn: publishContext[targetNamesKey], // NQN for NVMe + TargetAddress: publishContext[discoveryIPsKey], // Target IP + TargetPort: publishContext[targetPortKey], // Target port (default 4420) + ConnectionMode: defaultConnectionMode, + } + + // Cleanup any stale device existing before stage + device, _ := driver.chapiDriver.GetDevice(volume) + if device != nil { + device.TargetScope = volume.TargetScope + err := driver.chapiDriver.DeleteDevice(device) + if err != nil { + log.Warnf("Failed to cleanup stale NVMe device %s before staging, err %s", device.AltFullPathName, err.Error()) + } + } + + // Create NVMe Device + devices, err := driver.chapiDriver.CreateDevices([]*model.Volume{volume}) + if err != nil { + log.Errorf("Failed to create NVMe device from publish info. Error: %s", err.Error()) + return nil, err + } + if len(devices) == 0 { + log.Errorf("Failed to get the NVMe device just created using the volume %+v", volume) + return nil, fmt.Errorf("unable to find the NVMe device for volume %+v", volume) + } + + return devices[0], nil + } + // TODO: Enhance CHAPI to work with a PublishInfo object rather than a volume discoveryIps := strings.Split(publishContext[discoveryIPsKey], ",") @@ -2144,7 +2181,7 @@ func (driver *Driver) NodeGetInfo(ctx context.Context, req *csi.NodeGetInfoReque watcher, _ := util.InitializeWatcher(getNodeInfoFunc) // Add list of files /and directories to watch. The list contains // iSCSI , FC and CHAP Info and Networking config directories - list := []string{"/etc/sysconfig/network-scripts/", "/etc/sysconfig/network/", "/etc/iscsi/initiatorname.iscsi", "/etc/networks", "/etc/iscsi/iscsid.conf"} + list := []string{"/etc/sysconfig/network-scripts/", "/etc/sysconfig/network/", "/etc/iscsi/initiatorname.iscsi", "/etc/networks", "/etc/iscsi/iscsid.conf", "/etc/nvme/hostnqn"} watcher.AddWatchList(list) // Start event the watcher in a separate thread. go watcher.StartWatcher() @@ -2203,6 +2240,7 @@ func (driver *Driver) nodeGetInfo() (string, error) { var iqns []*string var wwpns []*string + var nqn *string for _, initiator := range initiators { if initiator.Type == iscsi { for i := 0; i < len(initiator.Init); i++ { @@ -2215,6 +2253,16 @@ func (driver *Driver) nodeGetInfo() (string, error) { } } + nvmeNqn, err := GetNvmeInitiator() + if err == nil && nvmeNqn != "" { + nqn = &nvmeNqn + } + + var nqns []*string + if nqn != nil { + nqns = append(nqns, nqn) + } + var cidrNetworks []*string for _, network := range networks { log.Infof("Processing network named %s with IpV4 CIDR %s", network.Name, network.CidrNetwork) @@ -2230,6 +2278,7 @@ func (driver *Driver) nodeGetInfo() (string, error) { Iqns: iqns, Networks: cidrNetworks, Wwpns: wwpns, + Nqns: nqns, } nodeID, err := driver.flavor.LoadNodeInfo(node) diff --git a/pkg/driver/utils.go b/pkg/driver/utils.go index 6268ecc1..6b1d3f00 100644 --- a/pkg/driver/utils.go +++ b/pkg/driver/utils.go @@ -126,3 +126,12 @@ func removeDataFile(dirPath string, fileName string) error { func isValidIP(ip string) bool { return ip != "" && net.ParseIP(ip) != nil } +// GetNvmeInitiator returns the NVMe host NQN as a string +func GetNvmeInitiator() (string, error) { + data, err := ioutil.ReadFile("/etc/nvme/hostnqn") + if err != nil { + return "", err + } + nqn := strings.TrimSpace(string(data)) + return nqn, nil +} \ No newline at end of file diff --git a/pkg/flavor/kubernetes/flavor.go b/pkg/flavor/kubernetes/flavor.go index 5e6dafe0..b9007203 100644 --- a/pkg/flavor/kubernetes/flavor.go +++ b/pkg/flavor/kubernetes/flavor.go @@ -239,13 +239,18 @@ func (flavor *Flavor) LoadNodeInfo(node *model.Node) (string, error) { nodeInfo.Spec.WWPNs = wwpnsFromNode updateNodeRequired = true } + nqnsFromNode := getNqnsFromNode(node) + if !reflect.DeepEqual(nodeInfo.Spec.NQNs, nqnsFromNode) { + nodeInfo.Spec.NQNs = nqnsFromNode + updateNodeRequired = true + } if !updateNodeRequired { // no update needed to existing CRD return node.UUID, nil } - log.Infof("updating Node %s with iqns %v wwpns %v networks %v", - nodeInfo.Name, nodeInfo.Spec.IQNs, nodeInfo.Spec.WWPNs, nodeInfo.Spec.Networks) + log.Infof("updating Node %s with iqns %v wwpns %v networks %v nqns %v", + nodeInfo.Name, nodeInfo.Spec.IQNs, nodeInfo.Spec.WWPNs, nodeInfo.Spec.Networks, nodeInfo.Spec.NQNs) _, err := flavor.crdClient.StorageV1().HPENodeInfos().Update(nodeInfo) if err != nil { log.Errorf("Error updating the node %s - %s\n", nodeInfo.Name, err.Error()) @@ -262,6 +267,7 @@ func (flavor *Flavor) LoadNodeInfo(node *model.Node) (string, error) { IQNs: getIqnsFromNode(node), Networks: getNetworksFromNode(node), WWPNs: getWwpnsFromNode(node), + NQNs: getNqnsFromNode(node), }, } @@ -303,6 +309,14 @@ func getNetworksFromNode(node *model.Node) []string { return networks } +func getNqnsFromNode(node *model.Node) []string { + var nqns []string + for i := 0; i < len(node.Nqns); i++ { + nqns = append(nqns, *node.Nqns[i]) + } + return nqns +} + // UnloadNodeInfo remove the HPENodeInfo from the list of CRDs func (flavor *Flavor) UnloadNodeInfo() { log.Tracef(">>>>>> UnloadNodeInfo with name %s", flavor.nodeName) @@ -353,12 +367,17 @@ func (flavor *Flavor) GetNodeInfo(nodeID string) (*model.Node, error) { for i := range wwpns { wwpns[i] = &nodeInfo.Spec.WWPNs[i] } + nqns := make([]*string, len(nodeInfo.Spec.NQNs)) + for i := range nqns { + nqns[i] = &nodeInfo.Spec.NQNs[i] + } node := &model.Node{ Name: nodeInfo.ObjectMeta.Name, UUID: nodeInfo.Spec.UUID, Iqns: iqns, Networks: networks, Wwpns: wwpns, + Nqns: nqns, } return node, nil diff --git a/vendor/github.com/hpe-storage/common-host-libs/linux/device.go b/vendor/github.com/hpe-storage/common-host-libs/linux/device.go index 175a8874..4cddcf4c 100644 --- a/vendor/github.com/hpe-storage/common-host-libs/linux/device.go +++ b/vendor/github.com/hpe-storage/common-host-libs/linux/device.go @@ -49,6 +49,7 @@ const ( // lrwxrwxrwx 1 root root 0 Mar 8 16:51 sdg -> ../devices/platform/host4/session2/target4:0:0/4:0:0:2/block/sdg deviceByHctlPatternFmt = ".*%s:%s:%s:%s.*block/(?P.*)" lunNotSupportedErr = "LOGICAL UNIT NOT SUPPORTED" + nvmeDevicePattern = "nvme[0-9]+n[0-9]+" ) var ( @@ -425,7 +426,7 @@ func rescanLoginVolume(volume *model.Volume) error { secondaryVolObj.LunID = strconv.Itoa(int(secondaryLunInfo.LunID)) secondaryVolObj.Iqns = secondaryLunInfo.TargetNames secondaryVolObj.TargetScope = volume.TargetScope - secondaryVolObj.DiscoveryIPs = secondaryLunInfo.DiscoveryIPs + secondaryVolObj.DiscoveryIPs = secondaryLunInfo.IscsiAccessInfo.DiscoveryIPs secondaryVolObj.Chap = volume.Chap secondaryVolObj.ConnectionMode = volume.ConnectionMode secondaryVolObj.SerialNumber = volume.SerialNumber @@ -450,7 +451,13 @@ func rescanLoginVolumeForBackend(volObj *model.Volume) error { if err != nil { return err } - } else { + } else if strings.EqualFold(volObj.AccessProtocol, "nvmetcp") { + // NVMe over TCP volume + err = HandleNvmeTcpDiscovery(volObj) + if err != nil { + return err + } + }else { // Check if client intends us to specifically login using multiple IP addresses(cloud volumes) if len(volObj.Networks) > 0 { // check if ifaces are created and enable port binding @@ -1094,6 +1101,17 @@ func GetDeviceFromVolume(vol *model.Volume) (*model.Device, error) { if err != nil { return nil, err } + + if len(devices) > 0 { + return devices[0], nil + } + + // Try NVMe device lookup if not found above + nvmeDev, err := GetNvmeDeviceFromNamespace(vol.SerialNumber) + if err == nil && nvmeDev != nil { + log.Debugf("Found NVMe device: %+v", nvmeDev) + return nvmeDev, nil + } if len(devices) == 0 { return nil, fmt.Errorf("unable to find device matching volume serial number %s", vol.SerialNumber) } @@ -1426,3 +1444,27 @@ func isMappedLuksDevice(devPath string) (bool, error) { return false, err } } + +// GetNvmeDeviceFromNamespace returns NVMe device path for given namespace or serial +func GetNvmeDeviceFromNamespace(serialOrNamespace string) (*model.Device, error) { + nvmeRoot := "/dev/" + files, err := ioutil.ReadDir(nvmeRoot) + if err != nil { + return nil, err + } + nvmeRegex := regexp.MustCompile(nvmeDevicePattern) + for _, f := range files { + if nvmeRegex.MatchString(f.Name()) { + // Optionally, check namespace/serial via sysfs + sysfsSerialPath := fmt.Sprintf("/sys/class/block/%s/serial", f.Name()) + serial, _ := util.FileReadFirstLine(sysfsSerialPath) + if serialOrNamespace == f.Name() || serialOrNamespace == serial { + return &model.Device{ + Pathname: nvmeRoot + f.Name(), + SerialNumber: serial, + }, nil + } + } + } + return nil, fmt.Errorf("NVMe device not found for %s", serialOrNamespace) +} \ No newline at end of file diff --git a/vendor/github.com/hpe-storage/common-host-libs/linux/initiator.go b/vendor/github.com/hpe-storage/common-host-libs/linux/initiator.go index e2f1e773..1858b32c 100644 --- a/vendor/github.com/hpe-storage/common-host-libs/linux/initiator.go +++ b/vendor/github.com/hpe-storage/common-host-libs/linux/initiator.go @@ -4,6 +4,7 @@ package linux import ( "errors" + log "github.com/hpe-storage/common-host-libs/logger" "github.com/hpe-storage/common-host-libs/model" "github.com/hpe-storage/common-host-libs/util" @@ -32,15 +33,24 @@ func GetInitiators() ([]*model.Initiator, error) { if err != nil { log.Debug("Error getting FcInitiator: ", err) } + // Add NVMe initiator discovery + nvmeInits, err := getNvmeInitiators() + if err != nil { + log.Debug("Error getting NvmeInitiator: ", err) + } + if fcInits != nil { inits = append(inits, fcInits) } if iscsiInits != nil { inits = append(inits, iscsiInits) } + if nvmeInits != nil { + inits = append(inits, nvmeInits) + } - if fcInits == nil && iscsiInits == nil { - return nil, errors.New("iscsi and fc initiators not found") + if fcInits == nil && iscsiInits == nil && nvmeInits == nil { + return nil, errors.New("iscsi, fc, and nvme initiators not found") } log.Debug("initiators ", inits) @@ -94,3 +104,18 @@ func getFcInitiators() (fcInit *model.Initiator, err error) { } return fcInit, nil } + +func getNvmeInitiators() (init *model.Initiator, err error) { + log.Trace(">>>>> getNvmeInitiators") + defer log.Trace("<<<<< getNvmeInitiators") + + hostnqn, err := GetNvmeInitiator() + if err != nil { + log.Debugf("NVMe host NQN not found, assuming not an NVMe host") + return nil, nil + } + + initiators := []string{hostnqn} + init = &model.Initiator{Type: "nvmeotcp", Init: initiators} + return init, nil +} diff --git a/vendor/github.com/hpe-storage/common-host-libs/linux/iscsi.go b/vendor/github.com/hpe-storage/common-host-libs/linux/iscsi.go index d79f1da6..69506bc5 100644 --- a/vendor/github.com/hpe-storage/common-host-libs/linux/iscsi.go +++ b/vendor/github.com/hpe-storage/common-host-libs/linux/iscsi.go @@ -251,7 +251,7 @@ func HandleIscsiDiscovery(volume *model.Volume) (err error) { secondaryVolObj.LunID = strconv.Itoa(int(secondaryLunInfo.LunID)) secondaryVolObj.Iqns = secondaryLunInfo.TargetNames secondaryVolObj.TargetScope = volume.TargetScope - secondaryVolObj.DiscoveryIPs = secondaryLunInfo.DiscoveryIPs + secondaryVolObj.DiscoveryIPs = secondaryLunInfo.IscsiAccessInfo.DiscoveryIPs secondaryVolObj.Chap = volume.Chap secondaryVolObj.ConnectionMode = volume.ConnectionMode secondaryVolObj.SerialNumber = volume.SerialNumber diff --git a/vendor/github.com/hpe-storage/common-host-libs/linux/multipath.go b/vendor/github.com/hpe-storage/common-host-libs/linux/multipath.go index b3fd78fe..4bbaaf68 100644 --- a/vendor/github.com/hpe-storage/common-host-libs/linux/multipath.go +++ b/vendor/github.com/hpe-storage/common-host-libs/linux/multipath.go @@ -4,6 +4,8 @@ package linux import ( "fmt" + "io/ioutil" + "path/filepath" "regexp" "strings" "sync" @@ -219,6 +221,15 @@ func cleanupDeviceAndSlaves(dev *model.Device) (err error) { log.Tracef(">>>>> cleanupDeviceAndSlaves called for %+v", dev) defer log.Trace("<<<<< cleanupDeviceAndSlaves") + + // --- NVMe multipath handling --- + if dev != nil && dev.Pathname != "" && IsNvmeDevice(dev.Pathname) { + if err := HandleMultipathForDevice(dev); err != nil { + return err + } + } + // --- End NVMe multipath handling --- + isFC := isFibreChannelDevice(dev.Slaves) // disable queuing on multipath @@ -498,3 +509,40 @@ func multipathGetPathsOfDevice(dev *model.Device, needActivePath bool) (paths [] } return paths, nil } +// IsNvmeDevice returns true if the device path is an NVMe device (e.g., /dev/nvmeXnY) +func IsNvmeDevice(devPath string) bool { + base := filepath.Base(devPath) + matched, _ := regexp.MatchString(`^nvme\d+n\d+$`, base) + return matched +} + +// IsNvmeMultipathEnabled checks if NVMe native multipath is enabled for a device +func IsNvmeMultipathEnabled(devPath string) bool { + base := filepath.Base(devPath) + nvmeMpPath := filepath.Join("/sys/class/block", base, "nvme_multipath") + data, err := ioutil.ReadFile(nvmeMpPath) + if err != nil { + return false + } + return strings.TrimSpace(string(data)) == "Y" +} +// HandleMultipathForDevice handles multipath for both SCSI and NVMe devices +func HandleMultipathForDevice(dev *model.Device) error { + + if IsNvmeDevice(dev.Pathname) { + if IsNvmeMultipathEnabled(dev.Pathname) { + log.Debugf("NVMe native multipath is enabled for %s, skipping dm-multipath logic", dev.Pathname) + // All path management is handled by the NVMe subsystem + return nil + } + log.Debugf("NVMe device %s does not have native multipath enabled", dev.Pathname) + // Optional: fallback to dm-multipath for NVMe if required by your environment + // If not required, just return nil here + // Otherwise, call your existing dm-multipath logic below + // return handleDmMultipathForNvme(dev) + return nil + } + // Existing dm-multipath logic for SCSI/iSCSI devices + // return handleDmMultipathForScsi(dev) + return nil +} \ No newline at end of file diff --git a/vendor/github.com/hpe-storage/common-host-libs/linux/nvme.go b/vendor/github.com/hpe-storage/common-host-libs/linux/nvme.go new file mode 100644 index 00000000..87266abe --- /dev/null +++ b/vendor/github.com/hpe-storage/common-host-libs/linux/nvme.go @@ -0,0 +1,197 @@ +// Copyright 2025 Hewlett Packard Enterprise Development LP + +package linux + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "regexp" + "strings" + "time" + + log "github.com/hpe-storage/common-host-libs/logger" + "github.com/hpe-storage/common-host-libs/model" + "github.com/hpe-storage/common-host-libs/util" +) + +const ( + nvmecmd = "nvme" + nvmeConnectCmd = "nvme connect" + nvmeDisconnectCmd = "nvme disconnect" + nvmeListCmd = "nvme list" + nvmeListSubsysCmd = "nvme list-subsys" + defaultNvmePort = "4420" + nvmeHostPathFormat = "/sys/class/nvme/" + nvmeNamespacePattern = "nvme[0-9]+n[0-9]+" + nvmeHostPath = "/etc/nvme/hostnqn" +) + +// GetNvmeInitiator gets the NVMe host NQN +func GetNvmeInitiator() (string, error) { + // Read from /etc/nvme/hostnqn or generate one + hostnqn, err := util.FileReadFirstLine(nvmeHostPath) + if err != nil { + log.Debugf("Could not read hostnqn from %s, generating one", nvmeHostPath) + // Generate hostnqn using nvme gen-hostnqn + args := []string{"gen-hostnqn"} + hostnqn, _, err = util.ExecCommandOutput(nvmecmd, args) + if err != nil { + return "", err + } + hostnqn = strings.TrimSpace(hostnqn) + } + return hostnqn, nil +} + +// ApplyNvmeTcpTuning applies recommended sysctl and module settings for NVMe over TCP +func ApplyNvmeTcpTuning() error { + var tuningErrors []string + + // Example: Increase network buffer sizes for high throughput + if err := setSysctl("net.core.rmem_max", "16777216"); err != nil { + tuningErrors = append(tuningErrors, err.Error()) + } + if err := setSysctl("net.core.wmem_max", "16777216"); err != nil { + tuningErrors = append(tuningErrors, err.Error()) + } + + // Example: Set NVMe core parameters (if needed) + if err := setKernelParam("/sys/module/nvme_core/parameters/multipath", "Y"); err != nil { + tuningErrors = append(tuningErrors, err.Error()) + } + + // Add more NVMe/TCP-specific tuning as needed... + + if len(tuningErrors) > 0 { + return fmt.Errorf("NVMe TCP tuning errors: %s", strings.Join(tuningErrors, "; ")) + } + return nil +} + +func setSysctl(key, value string) error { + cmd := fmt.Sprintf("sysctl -w %s=%s", key, value) + out, _, err := util.ExecCommandOutput("sh", []string{"-c", cmd}) + if err != nil { + return fmt.Errorf("failed to set %s: %v (%s)", key, err, out) + } + return nil +} + +func setKernelParam(path, value string) error { + f, err := os.OpenFile(path, os.O_WRONLY, 0) + if err != nil { + return fmt.Errorf("failed to open %s: %v", path, err) + } + defer f.Close() + if _, err := f.WriteString(value); err != nil { + return fmt.Errorf("failed to write %s to %s: %v", value, path, err) + } + return nil +} + +// ConnectNvmeTarget connects to an NVMe over TCP target +func ConnectNvmeTarget(target *model.NvmeTarget) error { + + args := []string{ + "connect", + "-t", "tcp", + "-n", target.NQN, + "-a", target.Address, + "-s", target.Port, + } + + _, _, err := util.ExecCommandOutput(nvmecmd, args) + return err +} + +// DisconnectNvmeTarget disconnects from an NVMe target +func DisconnectNvmeTarget(target *model.NvmeTarget) error { + args := []string{ + "disconnect", + "-n", target.NQN, + } + + _, _, err := util.ExecCommandOutput(nvmecmd, args) + return err +} + +// RescanNvme performs NVMe namespace rescan +func RescanNvme() error { + // NVMe typically doesn't require explicit rescanning like SCSI + // The kernel automatically detects new namespaces + return nil +} +// HandleNvmeTcpDiscovery performs NVMe/TCP connection and device verification for a volume. +func HandleNvmeTcpDiscovery(volume *model.Volume) error { + log.Tracef(">>>>> HandleNvmeTcpDiscovery for volume %s", volume.SerialNumber) + defer log.Trace("<<<<< HandleNvmeTcpDiscovery") + + // 1. Apply NVMe/TCP tuning recommendations + if err := ApplyNvmeTcpTuning(); err != nil { + log.Warnf("Failed to apply NVMe TCP tuning: %v", err) + // Continue even if tuning fails + } + + // 2. Prepare NVMe target info + target := &model.NvmeTarget{ + NQN: volume.Nqn, + Address: volume.TargetAddress, + Port: volume.TargetPort, + } + + // 3. Connect to NVMe target + if err := ConnectNvmeTarget(target); err != nil { + return fmt.Errorf("failed to connect to NVMe target: %v", err) + } + + // 4. Optionally, verify device presence (wait for /dev/nvmeXnY) + found := false + for i := 0; i < 10; i++ { + devices, _ := FindNvmeDevices(volume.SerialNumber) + if len(devices) > 0 { + found = true + break + } + time.Sleep(1 * time.Second) + } + if !found { + return fmt.Errorf("NVMe device for serial %s not found after connect", volume.SerialNumber) + } + + return nil +} + +// FindNvmeDevices searches for NVMe devices matching the given serial number +func FindNvmeDevices(serialNumber string) ([]string, error) { + var devices []string + + // Scan /dev for nvme devices + files, err := ioutil.ReadDir("/dev") + if err != nil { + return nil, err + } + + nvmeRegex := regexp.MustCompile(`^nvme\d+n\d+$`) + for _, f := range files { + if nvmeRegex.MatchString(f.Name()) { + devicePath := filepath.Join("/dev", f.Name()) + + // Check serial number via sysfs + sysfsSerialPath := fmt.Sprintf("/sys/class/block/%s/serial", f.Name()) + if serial, err := util.FileReadFirstLine(sysfsSerialPath); err == nil { + if strings.TrimSpace(serial) == serialNumber { + devices = append(devices, devicePath) + } + } + + // Also check if the device name itself matches (for namespace matching) + if f.Name() == serialNumber { + devices = append(devices, devicePath) + } + } + } + + return devices, nil +} \ No newline at end of file diff --git a/vendor/github.com/hpe-storage/common-host-libs/model/types.go b/vendor/github.com/hpe-storage/common-host-libs/model/types.go index f049ccdd..d79a82b4 100644 --- a/vendor/github.com/hpe-storage/common-host-libs/model/types.go +++ b/vendor/github.com/hpe-storage/common-host-libs/model/types.go @@ -145,6 +145,14 @@ type IscsiTarget struct { Scope string // GST or VST } +// NvmeTarget struct +type NvmeTarget struct { + NQN string + Address string + Port string // 4420 (default NVMe over TCP port) + Scope string // GST or VST +} + // Device struct type Device struct { VolumeID string `json:"volume_id,omitempty"` @@ -159,6 +167,7 @@ type Device struct { Size int64 `json:"size,omitempty"` // size in MiB Slaves []string `json:"slaves,omitempty"` IscsiTargets []*IscsiTarget `json:"iscsi_target,omitempty"` + NvmeTargets []*NvmeTarget `json:"nvme_target,omitempty"` Hcils []string `json:"-"` // export it if needed TargetScope string `json:"target_scope,omitempty"` //GST="group", VST="volume" or empty(older array fiji etc), and no-op for FC State string `json:"state,omitempty"` // state of the device needed to verify the device is active @@ -190,6 +199,9 @@ type Volume struct { AccessProtocol string `json:"access_protocol,omitempty"` Iqn string `json:"iqn,omitempty"` // deprecated Iqns []string `json:"iqns,omitempty"` + Nqn string `json:"nqn,omitempty"` + TargetAddress string `json:"target_address,omitempty"` + TargetPort string `json:"target_port,omitempty"` DiscoveryIP string `json:"discovery_ip,omitempty"` // deprecated DiscoveryIPs []string `json:"discovery_ips,omitempty"` MountPoint string `json:"Mountpoint,omitempty"` @@ -201,6 +213,7 @@ type Volume struct { TargetScope string `json:"target_scope,omitempty"` //GST="group", VST="volume" or empty(older array fiji etc), and no-op for FC IscsiSessions []*IscsiSession `json:"iscsi_sessions,omitempty"` FcSessions []*FcSession `json:"fc_sessions,omitempty"` + NvmeSessions []*NvmeSession `json:"nvme_sessions,omitempty"` VolumeGroupId string `json:"volume_group_id"` SecondaryArrayDetails string `json:"secondary_array_details,omitempty"` UsedBytes int64 `json:"used_bytes,omitempty"` @@ -228,6 +241,11 @@ type IscsiSession struct { InitiatorNameLegacy string `json:"initiatorName,omitempty"` InitiatorIP string `json:"initiator_ip_addr,omitempty"` } +// NvmeSession info +type NvmeSession struct { + InitiatorNQN string `json:"initiator_nqn,omitempty"` + InitiatorIP string `json:"initiator_ip_addr,omitempty"` +} func (s FcSession) InitiatorWwpnStr() string { if s.InitiatorWwpnLegacy != "" { @@ -300,6 +318,7 @@ type BlockDeviceAccessInfo struct { LunID int32 `json:"lun_id,omitempty"` SecondaryBackendDetails IscsiAccessInfo + NvmetcpAccessInfo } // Information of LUN id, IQN, discovery IP's the secondary array @@ -309,9 +328,10 @@ type SecondaryBackendDetails struct { // Information of the each secondary array type SecondaryLunInfo struct { - LunID int32 `json:"lun_id,omitempty""` + LunID int32 `json:"lun_id,omitempty"` TargetNames []string `json:"target_names,omitempty"` IscsiAccessInfo + NvmetcpAccessInfo } // IscsiAccessInfo contains the fields necessary for iSCSI access @@ -321,6 +341,12 @@ type IscsiAccessInfo struct { ChapPassword string `json:"chap_password,omitempty"` } +// NvmetcpAccessInfo contains the fields necessary for NVMe/TCP access +type NvmetcpAccessInfo struct { + DiscoveryIPs []string `json:"discovery_ips,omitempty"` + TargetNames []string `json:"target_names,omitempty"` // NQN(s) + TargetPort string `json:"target_port,omitempty"` // e.g., 4420 +} // VirtualDeviceAccessInfo contains the required data to access a virtual device type VirtualDeviceAccessInfo struct { } @@ -398,6 +424,7 @@ type Node struct { Iqns []*string `json:"iqns,omitempty"` Networks []*string `json:"networks,omitempty"` Wwpns []*string `json:"wwpns,omitempty"` + Nqns []*string `json:"nqns,omitempty"` ChapUser string `json:"chap_user,omitempty"` ChapPassword string `json:"chap_password,omitempty"` AccessProtocol string `json:"access_protocol,omitempty"` diff --git a/vendor/github.com/hpe-storage/common-host-libs/tunelinux/nvme.go b/vendor/github.com/hpe-storage/common-host-libs/tunelinux/nvme.go new file mode 100644 index 00000000..8a8e7bae --- /dev/null +++ b/vendor/github.com/hpe-storage/common-host-libs/tunelinux/nvme.go @@ -0,0 +1,2 @@ +package tunelinux + diff --git a/vendor/github.com/hpe-storage/common-host-libs/util/volume.go b/vendor/github.com/hpe-storage/common-host-libs/util/volume.go index eb6ab8fc..f8a57616 100644 --- a/vendor/github.com/hpe-storage/common-host-libs/util/volume.go +++ b/vendor/github.com/hpe-storage/common-host-libs/util/volume.go @@ -4,6 +4,7 @@ package util import ( "encoding/json" + "github.com/hpe-storage/common-host-libs/logger" "github.com/hpe-storage/common-host-libs/model" ) @@ -70,7 +71,7 @@ func GetSecondaryArrayDiscoveryIps(details string) []string { numberOfSecondaryBackends := len(secondaryArrayDetails.PeerArrayDetails) var secondaryDiscoverIps []string for i := 0; i < numberOfSecondaryBackends; i++ { - for _, discoveryIpRetrieved := range secondaryArrayDetails.PeerArrayDetails[i].DiscoveryIPs { + for _, discoveryIpRetrieved := range secondaryArrayDetails.PeerArrayDetails[i].IscsiAccessInfo.DiscoveryIPs { secondaryDiscoverIps = append(secondaryDiscoverIps, discoveryIpRetrieved) } } diff --git a/vendor/github.com/hpe-storage/k8s-custom-resources/pkg/apis/hpestorage/v1/types.go b/vendor/github.com/hpe-storage/k8s-custom-resources/pkg/apis/hpestorage/v1/types.go index 98112a44..4d0e0bc0 100644 --- a/vendor/github.com/hpe-storage/k8s-custom-resources/pkg/apis/hpestorage/v1/types.go +++ b/vendor/github.com/hpe-storage/k8s-custom-resources/pkg/apis/hpestorage/v1/types.go @@ -32,6 +32,7 @@ type HPENodeInfoSpec struct { IQNs []string `json:"iqns,omitempty"` Networks []string `json:"networks,omitempty"` WWPNs []string `json:"wwpns,omitempty"` + NQNs []string `json:"nqns,omitempty"` ChapUser string `json:"chapUser,omitempty"` ChapPassword string `json:"chapPassword,omitempty"` }