Add file share export paths to volume attributes so that CSI provisioner can add them as PVC annotations

Author: vdkotkar

pkg/csi/service/common/constants.go

@@ -144,9 +144,18 @@ const (
 	// UnknownVolumeType is assigned to CNS volumes whose type couldn't be determined.
 	UnknownVolumeType = "UNKNOWN"
 
+	// Nfsv3AccessPointKey is the key for NFSv3 access point.
+	Nfsv3AccessPointKey = "NFSv3"
+
 	// Nfsv4AccessPointKey is the key for NFSv4 access point.
 	Nfsv4AccessPointKey = "NFSv4.1"
 
+	// Nfsv3ExportPathAnnotationKey specifies the NFSv3 export path annotation key on PVC
+	Nfsv3ExportPathAnnotationKey = "csi.vsphere.exportpath.nfs3"
+
+	// Nfsv4ExportPathAnnotationKey specifies the NFSv4.1 export path annotation key on PVC
+	Nfsv4ExportPathAnnotationKey = "csi.vsphere.exportpath.nfs41"
+
 	// Nfsv4AccessPoint is the access point of file volume.
 	Nfsv4AccessPoint = "Nfsv4AccessPoint"
 

pkg/csi/service/wcp/controller.go

@@ -1503,6 +1503,31 @@ func (c *controller) createFileVolume(ctx context.Context, req *csi.CreateVolume
 	attributes := make(map[string]string)
 	attributes[common.AttributeDiskType] = common.DiskTypeFileVolume
 
+	// Return NFSv3 and NFSv4.1 export paths in volume context attributes.
+	// CSI provisioner will add these export paths as PVC annotations.
+	// Make a QueryVolume call to fetch file share export paths.
+	querySelection := cnstypes.CnsQuerySelection{
+		Names: []string{
+			string(cnstypes.QuerySelectionNameTypeBackingObjectDetails),
+		},
+	}
+	volume, err := common.QueryVolumeByID(ctx, c.manager.VolumeManager,
+		volumeInfo.VolumeID.Id, &querySelection)
+	if err != nil {
+		log.Errorf("error while performing QueryVolume on volume %s, error: %+v",
+			volumeInfo.VolumeID.Id, err)
+	}
+	vSANFileBackingDetails := volume.BackingObjectDetails.(*cnstypes.CnsVsanFileShareBackingDetails)
+	for _, kv := range vSANFileBackingDetails.AccessPoints {
+		if kv.Key == common.Nfsv3AccessPointKey {
+			attributes[common.Nfsv3ExportPathAnnotationKey] = kv.Value
+		} else if kv.Key == common.Nfsv4AccessPointKey {
+			attributes[common.Nfsv4ExportPathAnnotationKey] = kv.Value
+		}
+	}
+	log.Debugf("Access point details for volume: %s, namespace: %s are %+v", req.Name,
+		req.Parameters[common.AttributePvcNamespace], attributes)
+
 	resp := &csi.CreateVolumeResponse{
 		Volume: &csi.Volume{
 			VolumeId:      volumeInfo.VolumeID.Id,

pkg/syncer/fullsync.go

@@ -224,6 +224,32 @@ func CsiFullSync(ctx context.Context, metadataSyncer *metadataSyncInformer, vc s
 		}
 	}
 
+	// Iterate over all the file volume PVCs and check if file share export paths are added as annotations
+	// on it. If not added, then add file share export path annotations on such PVCs.
+	if metadataSyncer.clusterFlavor == cnstypes.CnsClusterFlavorWorkload {
+		k8sClient, err := k8sNewClient(ctx)
+		if err != nil {
+			log.Errorf("FullSync for VC %s: Failed to create kubernetes client. Err: %+v", vc, err)
+			return err
+		}
+		for _, pv := range k8sPVs {
+			if IsFileVolume(pv) {
+				if pvc, ok := pvToPVCMap[pv.Name]; ok {
+					// Check if file share export path is available as annotation on PVC
+					if pvc.ObjectMeta.Annotations[common.Nfsv3ExportPathAnnotationKey] == "" ||
+						pvc.ObjectMeta.Annotations[common.Nfsv4ExportPathAnnotationKey] == "" {
+						err = setFileShareAnnotationsOnPVC(ctx, k8sClient, metadataSyncer.volumeManager, pvc)
+						if err != nil {
+							log.Errorf("FullSync for VC %s: Failed to add file share export path annotations "+
+								"for PVC %s, Err: %v. Continuing for other PVCs.", pvc.Name, err)
+							continue
+						}
+					}
+				}
+			}
+		}
+	}
+
 	var queryAllResult *cnstypes.CnsQueryResult
 	if metadataSyncer.configInfo.Cfg.Global.ClusterID != "" {
 		// Cluster ID is removed from vSphere Config Secret post 9.0 release in Supervisor
@@ -630,6 +656,60 @@ func patchVolumeAccessibleTopologyToPVC(ctx context.Context, k8sClient clientset
 	return nil
 }
 
+// setFileShareAnnotationsOnPVC sets file share export paths as annotation on file volume PVC if it
+// is not already added. In case of upgrade from old version to VC 9.1 compatible CSI driver, this
+// will add these annotations on all existing file volume PVCs.
+func setFileShareAnnotationsOnPVC(ctx context.Context, k8sClient clientset.Interface,
+	volumeManager volumes.Manager, pvc *v1.PersistentVolumeClaim) error {
+	log := logger.GetLogger(ctx)
+	log.Infof("setFileShareAnnotationsOnPVC: Setting file share annotation for PVC: %q, namespace: %q",
+		pvc.Name, pvc.Namespace)
+
+	namespacedPvcName := pvc.Namespace + "/" + pvc.Name
+	volumeId, ok := commonco.ContainerOrchestratorUtility.GetVolumeIDFromPVCName(namespacedPvcName)
+	if !ok {
+		errmsg := fmt.Sprintf("failed to get volumeId for PVC: %q, namespace: %q", pvc.Name, pvc.Namespace)
+		log.Errorf("setFileShareAnnotationsOnPVC: %s", errmsg)
+		return errors.New(errmsg)
+	}
+
+	// Make a QueryVolume call to fetch file share export paths.
+	querySelection := cnstypes.CnsQuerySelection{
+		Names: []string{
+			string(cnstypes.QuerySelectionNameTypeBackingObjectDetails),
+		},
+	}
+	volume, err := common.QueryVolumeByID(ctx, volumeManager, volumeId, &querySelection)
+	if err != nil {
+		log.Errorf("setFileShareAnnotationsOnPVC: Error while performing QueryVolume on volume %s, Err: %+v",
+			volumeId, err)
+	}
+	vSANFileBackingDetails := volume.BackingObjectDetails.(*cnstypes.CnsVsanFileShareBackingDetails)
+	accessPoints := make(map[string]string)
+	for _, kv := range vSANFileBackingDetails.AccessPoints {
+		if kv.Key == common.Nfsv3AccessPointKey {
+			pvc.Annotations[common.Nfsv3ExportPathAnnotationKey] = kv.Value
+		} else if kv.Key == common.Nfsv4AccessPointKey {
+			pvc.Annotations[common.Nfsv4ExportPathAnnotationKey] = kv.Value
+		}
+		accessPoints[kv.Key] = kv.Value
+	}
+	log.Debugf("setFileShareAnnotationsOnPVC: Access point details for PVC: %q, namespace: %q are %+v",
+		pvc.Name, pvc.Namespace, accessPoints)
+
+	// Update PVC to add annotation on it
+	pvc, err = k8sClient.CoreV1().PersistentVolumeClaims(pvc.Namespace).Update(ctx, pvc,
+		metav1.UpdateOptions{})
+	if err != nil {
+		log.Errorf("setFileShareAnnotationsOnPVC: Error updating PVC %q in namespace %q, Err: %v",
+			pvc.Name, pvc.Namespace, err)
+		return err
+	}
+	log.Infof("setFileShareAnnotationsOnPVC: Added file share export paths annotation successfully on PVC %q, "+
+		"namespce %q", pvc.Name, pvc.Namespace)
+	return nil
+}
+
 // cleanUpVolumeInfoCrDeletionMap removes volumes from the VolumeInfo CR deletion map
 // if the volume is present in the K8s cluster.
 // This may happen if is not yet created PV by the time full sync

pkg/syncer/syncer_test.go

@@ -1013,6 +1013,15 @@ func runTestCsiFullSync_WorkloadCluster(t *testing.T) {
 		commonco.ContainerOrchestratorUtility = originalCO
 	}()
 
+	// Store the original K8sNewclient to restore later
+	origK8sClient := k8sNewClient
+	defer func() {
+		k8sNewClient = origK8sClient
+	}()
+	k8sNewClient = func(ctx context.Context) (clientset.Interface, error) {
+		return k8sclient, nil
+	}
+
 	// Create a WORKLOAD cluster metadataSyncer (note: volumeManagers map is NOT populated)
 	// This simulates the real WORKLOAD cluster setup where only volumeManager is set
 	workloadMetadataSyncer := &metadataSyncInformer{