CON-3876 - NVMEoTCP support for csi-driver and host-libs

pkg/driver/controller_server.go

@@ -924,7 +924,9 @@ func (driver *Driver) controllerPublishVolume(
                 requestedAccessProtocol = iscsi
         } else if requestedAccessProtocol == "fc" {
                 requestedAccessProtocol = fc
-        }
+        } else if requestedAccessProtocol == "nvmetcp" {
+		requestedAccessProtocol = "nvmetcp"
+	}
 
 	if existingNode != nil {
 		log.Tracef("CSP has already been notified about the node with ID %s and UUID %s", existingNode.ID, existingNode.UUID)
@@ -957,12 +959,33 @@ func (driver *Driver) controllerPublishVolume(
 
 	// TODO: add any additional info necessary to mount the device
 	publishContext := map[string]string{}
-	publishContext[serialNumberKey] = publishInfo.SerialNumber
-	publishContext[accessProtocolKey] = publishInfo.AccessInfo.BlockDeviceAccessInfo.AccessProtocol
-	publishContext[targetNamesKey] = strings.Join(publishInfo.AccessInfo.BlockDeviceAccessInfo.TargetNames, ",")
-	publishContext[targetScopeKey] = requestedTargetScope
-	publishContext[lunIDKey] = strconv.Itoa(int(publishInfo.AccessInfo.BlockDeviceAccessInfo.LunID))
+
+	// NVMe over TCP support
+    if strings.EqualFold(publishInfo.AccessInfo.BlockDeviceAccessInfo.AccessProtocol, "nvmetcp") {
+        publishContext[serialNumberKey] = publishInfo.SerialNumber
+        publishContext[accessProtocolKey] = "nvmetcp"
+        // NQN, target address, and port for NVMe/TCP
+        if len(publishInfo.AccessInfo.BlockDeviceAccessInfo.TargetNames) > 0 {
+            publishContext[targetNamesKey] = publishInfo.AccessInfo.BlockDeviceAccessInfo.TargetNames[0]
+        }
+        if len(publishInfo.AccessInfo.BlockDeviceAccessInfo.NvmetcpAccessInfo.DiscoveryIPs) > 0 {
+            publishContext[discoveryIPsKey] = publishInfo.AccessInfo.BlockDeviceAccessInfo.NvmetcpAccessInfo.DiscoveryIPs[0]
+        }
+        if publishInfo.AccessInfo.BlockDeviceAccessInfo.NvmetcpAccessInfo.TargetPort != "" {
+            publishContext[targetPortKey] = publishInfo.AccessInfo.BlockDeviceAccessInfo.NvmetcpAccessInfo.TargetPort
+        } else {
+            publishContext[targetPortKey] = "4420" // default NVMe/TCP port
+        }
+
+    }else{
+		publishContext[serialNumberKey] = publishInfo.SerialNumber
+		publishContext[accessProtocolKey] = publishInfo.AccessInfo.BlockDeviceAccessInfo.AccessProtocol
+		publishContext[targetNamesKey] = strings.Join(publishInfo.AccessInfo.BlockDeviceAccessInfo.TargetNames, ",")
+		publishContext[targetScopeKey] = requestedTargetScope
+		publishContext[lunIDKey] = strconv.Itoa(int(publishInfo.AccessInfo.BlockDeviceAccessInfo.LunID))
 
+	}
+
 	// Start of population of target array details
 	if publishInfo.AccessInfo.BlockDeviceAccessInfo.SecondaryBackendDetails.PeerArrayDetails != nil {
 		secondaryArrayMarshalledStr, err := json.Marshal(&publishInfo.AccessInfo.BlockDeviceAccessInfo.SecondaryBackendDetails)

pkg/driver/node_server.go

@@ -41,6 +41,8 @@ var (
 
 const (
 	fileHostIPKey = "hostIP"
+	nvmetcp = "nvmetcp"
+    	targetPortKey = "targetPort"
 )
 
 var isWatcherEnabled = false
@@ -572,6 +574,41 @@ func (driver *Driver) setupDevice(
 	log.Tracef(">>>>> setupDevice, volumeID: %s, publishContext: %v", volumeID, log.MapScrubber(publishContext))
 	defer log.Trace("<<<<< setupDevice")
 
+	// Handle NVMe over TCP volumes
+    if publishContext[accessProtocolKey] == "nvmetcp" {
+        volume := &model.Volume{
+            SerialNumber:   publishContext[serialNumberKey],
+            AccessProtocol: publishContext[accessProtocolKey],
+            Nqn:           publishContext[targetNamesKey], // NQN for NVMe
+            TargetAddress: publishContext[discoveryIPsKey], // Target IP
+            TargetPort:    publishContext[targetPortKey],   // Target port (default 4420)
+            ConnectionMode: defaultConnectionMode,
+        }
+
+        // Cleanup any stale device existing before stage
+        device, _ := driver.chapiDriver.GetDevice(volume)
+        if device != nil {
+            device.TargetScope = volume.TargetScope
+            err := driver.chapiDriver.DeleteDevice(device)
+            if err != nil {
+                log.Warnf("Failed to cleanup stale NVMe device %s before staging, err %s", device.AltFullPathName, err.Error())
+            }
+        }
+
+        // Create NVMe Device
+        devices, err := driver.chapiDriver.CreateDevices([]*model.Volume{volume})
+        if err != nil {
+            log.Errorf("Failed to create NVMe device from publish info. Error: %s", err.Error())
+            return nil, err
+        }
+        if len(devices) == 0 {
+            log.Errorf("Failed to get the NVMe device just created using the volume %+v", volume)
+            return nil, fmt.Errorf("unable to find the NVMe device for volume %+v", volume)
+        }
+
+        return devices[0], nil
+    }
+
 	// TODO: Enhance CHAPI to work with a PublishInfo object rather than a volume
 
 	discoveryIps := strings.Split(publishContext[discoveryIPsKey], ",")
@@ -2144,7 +2181,7 @@ func (driver *Driver) NodeGetInfo(ctx context.Context, req *csi.NodeGetInfoReque
 		watcher, _ := util.InitializeWatcher(getNodeInfoFunc)
 		// Add list of files /and directories to watch. The list contains
 		// iSCSI , FC and CHAP Info and Networking config directories
-		list := []string{"/etc/sysconfig/network-scripts/", "/etc/sysconfig/network/", "/etc/iscsi/initiatorname.iscsi", "/etc/networks", "/etc/iscsi/iscsid.conf"}
+		list := []string{"/etc/sysconfig/network-scripts/", "/etc/sysconfig/network/", "/etc/iscsi/initiatorname.iscsi", "/etc/networks", "/etc/iscsi/iscsid.conf", "/etc/nvme/hostnqn"}
 		watcher.AddWatchList(list)
 		// Start event the watcher in a separate thread.
 		go watcher.StartWatcher()
@@ -2203,6 +2240,7 @@ func (driver *Driver) nodeGetInfo() (string, error) {
 
 	var iqns []*string
 	var wwpns []*string
+	var nqn *string
 	for _, initiator := range initiators {
 		if initiator.Type == iscsi {
 			for i := 0; i < len(initiator.Init); i++ {
@@ -2215,6 +2253,16 @@ func (driver *Driver) nodeGetInfo() (string, error) {
 		}
 	}
 
+	nvmeNqn, err := GetNvmeInitiator()
+	if err == nil && nvmeNqn != "" {
+		nqn = &nvmeNqn
+	}
+
+	var nqns []*string
+	if nqn != nil {
+		nqns = append(nqns, nqn)
+	}
+
 	var cidrNetworks []*string
 	for _, network := range networks {
 		log.Infof("Processing network named %s with IpV4 CIDR %s", network.Name, network.CidrNetwork)
@@ -2230,6 +2278,7 @@ func (driver *Driver) nodeGetInfo() (string, error) {
 		Iqns:     iqns,
 		Networks: cidrNetworks,
 		Wwpns:    wwpns,
+		Nqns:     nqns,
 	}
 
 	nodeID, err := driver.flavor.LoadNodeInfo(node)

pkg/driver/utils.go

@@ -126,3 +126,12 @@ func removeDataFile(dirPath string, fileName string) error {
 func isValidIP(ip string) bool {
 	return ip != "" && net.ParseIP(ip) != nil
 }
+// GetNvmeInitiator returns the NVMe host NQN as a string
+func GetNvmeInitiator() (string, error) {
+    data, err := ioutil.ReadFile("/etc/nvme/hostnqn")
+    if err != nil {
+        return "", err
+    }
+    nqn := strings.TrimSpace(string(data))
+    return nqn, nil
+}

pkg/flavor/kubernetes/flavor.go

@@ -239,13 +239,18 @@ func (flavor *Flavor) LoadNodeInfo(node *model.Node) (string, error) {
 			nodeInfo.Spec.WWPNs = wwpnsFromNode
 			updateNodeRequired = true
 		}
+		nqnsFromNode := getNqnsFromNode(node)
+		if !reflect.DeepEqual(nodeInfo.Spec.NQNs, nqnsFromNode) {
+			nodeInfo.Spec.NQNs = nqnsFromNode
+			updateNodeRequired = true
+		}
 
 		if !updateNodeRequired {
 			// no update needed to existing CRD
 			return node.UUID, nil
 		}
-		log.Infof("updating Node %s with iqns %v wwpns %v networks %v",
-			nodeInfo.Name, nodeInfo.Spec.IQNs, nodeInfo.Spec.WWPNs, nodeInfo.Spec.Networks)
+		log.Infof("updating Node %s with iqns %v wwpns %v networks %v nqns %v",
+			nodeInfo.Name, nodeInfo.Spec.IQNs, nodeInfo.Spec.WWPNs, nodeInfo.Spec.Networks, nodeInfo.Spec.NQNs)
 		_, err := flavor.crdClient.StorageV1().HPENodeInfos().Update(nodeInfo)
 		if err != nil {
 			log.Errorf("Error updating the node %s - %s\n", nodeInfo.Name, err.Error())
@@ -262,6 +267,7 @@ func (flavor *Flavor) LoadNodeInfo(node *model.Node) (string, error) {
 				IQNs:     getIqnsFromNode(node),
 				Networks: getNetworksFromNode(node),
 				WWPNs:    getWwpnsFromNode(node),
+				NQNs:     getNqnsFromNode(node),
 			},
 		}
 
@@ -303,6 +309,14 @@ func getNetworksFromNode(node *model.Node) []string {
 	return networks
 }
 
+func getNqnsFromNode(node *model.Node) []string {
+    var nqns []string
+    for i := 0; i < len(node.Nqns); i++ {
+        nqns = append(nqns, *node.Nqns[i])
+    }
+    return nqns
+}
+
 // UnloadNodeInfo remove the HPENodeInfo from the list of CRDs
 func (flavor *Flavor) UnloadNodeInfo() {
 	log.Tracef(">>>>>> UnloadNodeInfo with name %s", flavor.nodeName)
@@ -353,12 +367,17 @@ func (flavor *Flavor) GetNodeInfo(nodeID string) (*model.Node, error) {
 			for i := range wwpns {
 				wwpns[i] = &nodeInfo.Spec.WWPNs[i]
 			}
+			nqns := make([]*string, len(nodeInfo.Spec.NQNs))
+			for i := range nqns {
+				nqns[i] = &nodeInfo.Spec.NQNs[i]
+			}
 			node := &model.Node{
 				Name:     nodeInfo.ObjectMeta.Name,
 				UUID:     nodeInfo.Spec.UUID,
 				Iqns:     iqns,
 				Networks: networks,
 				Wwpns:    wwpns,
+				Nqns:     nqns,
 			}
 
 			return node, nil