Cherry pick commits from master - 2e42ee to 08a7bd8 (#3642)

* Initialize volume manager in WORKLOAD cluster mode (#3598)

Signed-off-by: Deepak Kinni <[email protected]>

* removing static-prov tests from cf (#3606)

* Adding vks testcases to EF column for n-1 and n-2 tkg versions (#3601)

* fix createSnapshotWithTransaction workflow (#3597)

* Surface NotSupported error message during AttachVolume call (#3595)

Signed-off-by: Deepak Kinni <[email protected]>

* vmservice - remove IP check if private network (#3610)

* Added wait time befoew querrying vm images (#3612)

* Updated controller-gen version (#3614)

* Fix incorrect volume attach error handling in CNSNodeVMAttachment reconciler (#3590)

* wcp-snapshot test fix (#3620)

* pq tag on block vanilla tests (#3619)

* Allow devops users to attach GC RWX file volume with VM service VMs (#3611)

* adding wcp-vsan-stretch-vmsvc-tests to pq (#3627)

* adding disruptive testcases to PQ UTS section (#3624)

* Remove validations for batch attach (#3622)

* Wait until topology annotation gets added on supervisor PVCs while adding node affinity rules on PVs in guest cluster (#3588)

* Addin vSAN Stretch TKG to UTS PQ (#3631)

* Ensure that during detach PVC has a unique entry in volumeStatus (#3630)

* Update vm-operator api version and parallelize sync volume api calls (#3604)

* Fix CnsVolumeOperationRequest update logic when transaction support is enabled (#3623)

Signed-off-by: Deepak Kinni <[email protected]>

* Following enhancement made in the list-view task monitoring (#3584)

Before recreating a ListView, destroy the old ListView to avoid leaks.
Sends error results to pending tasks without blocking - change made in reportErrorOnAllPendingTasks
Uses non-blocking send for task result channel in processTaskUpdate
TaskResult channel is now buffered (size 1) - change made in waitOnTask

* fixes (#3637)

---------

Signed-off-by: Deepak Kinni <[email protected]>
Co-authored-by: Deepak Kinni <[email protected]>
Co-authored-by: kavyashree-r <[email protected]>
Co-authored-by: sipriyaa <[email protected]>
Co-authored-by: Divyen Patel <[email protected]>
Co-authored-by: Satyanarayana Kolluri <[email protected]>
Co-authored-by: Raj Kumar Gupta <[email protected]>
Co-authored-by: skogta <[email protected]>
Co-authored-by: Vipul Kotkar <[email protected]>
Co-authored-by: nikhilbarge <[email protected]>

Author: 10 people

Makefile

@@ -135,7 +135,7 @@ ifeq (, $(shell which controller-gen))
 	CONTROLLER_GEN_TMP_DIR=$$(mktemp -d) ;\
 	cd $$CONTROLLER_GEN_TMP_DIR ;\
 	go mod init tmp ;\
-	go install sigs.k8s.io/controller-tools/cmd/controller-gen@v0.14.0 ;\
+	go install sigs.k8s.io/controller-tools/cmd/controller-gen@v0.19.0 ;\
 	rm -rf $$CONTROLLER_GEN_TMP_DIR ;\
 	}
 CONTROLLER_GEN=$(GOBIN)/controller-gen

go.mod

@@ -23,7 +23,7 @@ require (
 	github.com/pkg/sftp v1.13.6
 	github.com/prometheus/client_golang v1.22.0
 	github.com/stretchr/testify v1.10.0
-	github.com/vmware-tanzu/vm-operator/api v1.8.7-0.20250820184450-53a697d52f9c
+	github.com/vmware-tanzu/vm-operator/api v1.9.1-0.20250923172217-bf5a74e51c65
 	github.com/vmware-tanzu/vm-operator/external/byok v0.0.0-20250509154507-b93e51fc90fa
 	github.com/vmware/govmomi v0.52.0-alpha.0.0.20250807230438-0eee109f1f2c
 	go.uber.org/zap v1.27.0

go.sum

@@ -287,8 +287,8 @@ github.com/thecodeteam/gofsutil v0.1.2 h1:FL87mBzZeeuDMZm8hpYLFcYylQdq6bbm8UQ1oc
 github.com/thecodeteam/gofsutil v0.1.2/go.mod h1:7bDOpr2aMnmdm9RTdxBEeqdOr+8RpnQhsB/VUEI3DgM=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7otjonDflCTK0BCfls4SPy3NcCVb5dqqmbRknE=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk=
-github.com/vmware-tanzu/vm-operator/api v1.8.7-0.20250820184450-53a697d52f9c h1:+lECoxyxQcMsxU2CJ53YvaLEwqfGduS/3yQaLLRDYDo=
-github.com/vmware-tanzu/vm-operator/api v1.8.7-0.20250820184450-53a697d52f9c/go.mod h1:hkc/QZCSHcosWWMPS6VWWR12WenZcNE3BaTJ/8A8sNE=
+github.com/vmware-tanzu/vm-operator/api v1.9.1-0.20250923172217-bf5a74e51c65 h1:oby5iJHxU5KgtajXxT8B8VxUUPH20Zh0KTYlI+18AYs=
+github.com/vmware-tanzu/vm-operator/api v1.9.1-0.20250923172217-bf5a74e51c65/go.mod h1:nWTPpxfe4gHuuYuFcrs86+NMxfkqPk3a3IlvI8TCWak=
 github.com/vmware-tanzu/vm-operator/external/byok v0.0.0-20250509154507-b93e51fc90fa h1:4MKu14YJ7J54O6QKmT4ds5EUpysWLLtQRMff73cVkmU=
 github.com/vmware-tanzu/vm-operator/external/byok v0.0.0-20250509154507-b93e51fc90fa/go.mod h1:8tiuyYslzjLIUmOlXZuGKQdQP2ZgWGCVhVeyptmZYnk=
 github.com/vmware/govmomi v0.52.0-alpha.0.0.20250807230438-0eee109f1f2c h1:1nMVFr1CBMSNLLjsfx3QPfZ5k0R1/O29QX/A2X0w3RQ=

manifests/supervisorcluster/1.29/cns-csi.yaml

@@ -732,6 +732,9 @@ rules:
   - apiGroups: ["cns.vmware.com"]
     resources: ["cnsfileaccessconfigs"]
     verbs: ["get", "list", "update"]
+  - apiGroups: ["vmoperator.vmware.com"]
+    resources: ["virtualmachines"]
+    verbs: ["get", "list"]
 ---
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1

manifests/supervisorcluster/1.30/cns-csi.yaml

@@ -737,6 +737,9 @@ rules:
   - apiGroups: ["cns.vmware.com"]
     resources: ["cnsfileaccessconfigs"]
     verbs: ["get", "list", "update"]
+  - apiGroups: ["vmoperator.vmware.com"]
+    resources: ["virtualmachines"]
+    verbs: ["get", "list"]
 ---
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1

manifests/supervisorcluster/1.31/cns-csi.yaml

@@ -737,6 +737,9 @@ rules:
   - apiGroups: ["cns.vmware.com"]
     resources: ["cnsfileaccessconfigs"]
     verbs: ["get", "list", "update"]
+  - apiGroups: ["vmoperator.vmware.com"]
+    resources: ["virtualmachines"]
+    verbs: ["get", "list"]
 ---
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1

manifests/supervisorcluster/1.32/cns-csi.yaml

@@ -737,6 +737,9 @@ rules:
   - apiGroups: ["cns.vmware.com"]
     resources: ["cnsfileaccessconfigs"]
     verbs: ["get", "list", "update"]
+  - apiGroups: ["vmoperator.vmware.com"]
+    resources: ["virtualmachines"]
+    verbs: ["get", "list"]
 ---
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1

pkg/apis/cnsoperator/cnsnodevmbatchattachment/v1alpha1/cnsnodebatchvmattachment_types.go

@@ -29,6 +29,9 @@ const (
 	IndependentPersistent DiskMode = "independent_persistent"
 	// Changes are immediately and permanently written to the virtual disk.
 	Persistent DiskMode = "persistent"
+	// Changes to virtual disk are made to a redo log and discarded at power off.
+	// It is not affected by snapshots.
+	IndependentNonPersistent = "independent_nonpersistent"
 )
 
 // The sharing mode of the virtual disk.

pkg/common/cns-lib/volume/listview.go

@@ -324,13 +324,23 @@ func (l *ListViewImpl) listenToTaskUpdates() {
 		l.mu.Lock()
 		log.Infof("acquired lock before re-creating listview")
 		if recreateView {
+			if l.listView != nil {
+				destroyListviewErr := l.listView.Destroy(l.ctx)
+				if destroyListviewErr != nil {
+					// ignoring the error and re-creating the list view
+					log.Errorf("failed to destroy listview object. err: %v", destroyListviewErr)
+				} else {
+					log.Info("successfully destroyed existing listview")
+				}
+			}
 			log.Info("re-creating the listView object")
 			err := l.createListView(l.ctx, nil)
 			if err != nil {
 				log.Errorf("failed to create a ListView. error: %+v", err)
 				l.mu.Unlock()
 				continue
 			}
+			log.Info("successfully created listview")
 
 			filter = getListViewWaitFilter(l.listView)
 			l.waitForUpdatesContext, l.waitForUpdatesCancelFunc = context.WithCancel(context.Background())
@@ -395,12 +405,19 @@ func (l *ListViewImpl) listenToTaskUpdates() {
 
 // reportErrorOnAllPendingTasks returns failure to all pending tasks in the map in case of vc failure
 func (l *ListViewImpl) reportErrorOnAllPendingTasks(err error) {
+	log := logger.GetLogger(context.Background())
 	for _, taskDetails := range l.taskMap.GetAll() {
 		result := TaskResult{
 			TaskInfo: nil,
 			Err:      err,
 		}
-		taskDetails.ResultCh <- result
+		// Non-blocking send
+		select {
+		case taskDetails.ResultCh <- result:
+			log.Infof("reported error for task %+v", taskDetails.Reference)
+		default:
+			log.Warnf("failed to report error for task %+v: channel blocked", taskDetails.Reference)
+		}
 	}
 }
 
@@ -431,8 +448,16 @@ func (l *ListViewImpl) processTaskUpdate(prop types.PropertyChange) {
 		result.TaskInfo = &taskInfo
 		result.Err = nil
 	}
-
-	taskDetails.ResultCh <- result
+	// Use a non-blocking send to prevent deadlocks when multiple goroutines
+	// try to send to the same channel (e.g., due to duplicate task updates from vSphere)
+	select {
+	case taskDetails.ResultCh <- result:
+		log.Infof("Successfully sent task result for task %+v", taskInfo.Task)
+	default:
+		// Channel is full/blocked, which means another goroutine already sent the result
+		// This can happen when vSphere sends duplicate task update events
+		log.Warnf("result channel full for task %+v, ignoring duplicate update", taskInfo.Task)
+	}
 }
 
 // RemoveTasksMarkedForDeletion goes over the list of tasks in the map

pkg/common/cns-lib/volume/manager.go

@@ -827,9 +827,9 @@ func (m *defaultManager) waitOnTask(csiOpContext context.Context,
 			return nil, err
 		}
 	}
-	ch := make(chan TaskResult)
+	ch := make(chan TaskResult, 1)
 	err := m.listViewIf.AddTask(csiOpContext, taskMoRef, ch)
-	if errors.Unwrap(err) == ErrListViewTaskAddition {
+	if errors.Is(err, ErrListViewTaskAddition) {
 		return nil, logger.LogNewErrorf(log, "%s. err: %v", listviewAdditionError, err)
 	} else if err != nil {
 		// in case the task is not found in VC, we are returning a ManagedObjectNotFound error wrapped as a soap fault
@@ -1119,7 +1119,7 @@ func (m *defaultManager) AttachVolume(ctx context.Context,
 		}
 
 		volumeOperationRes := taskResult.GetCnsVolumeOperationResult()
-		if volumeOperationRes.Fault != nil {
+		if volumeOperationRes.Fault != nil && volumeOperationRes.Fault.Fault != nil {
 			faultType = ExtractFaultTypeFromVolumeResponseResult(ctx, volumeOperationRes)
 			_, isResourceInUseFault := volumeOperationRes.Fault.Fault.(*vim25types.ResourceInUse)
 			if isResourceInUseFault {
@@ -1133,6 +1133,37 @@ func (m *defaultManager) AttachVolume(ctx context.Context,
 					return diskUUID, "", nil
 				}
 			}
+
+			// Check if this is a CnsFault with NotSupported fault cause
+			if cnsFault, isCnsFault := volumeOperationRes.Fault.Fault.(*cnstypes.CnsFault); isCnsFault {
+				if cnsFault.FaultCause != nil {
+					notSupportedFault, isNotSupportedFault := cnsFault.FaultCause.Fault.(*vim25types.NotSupported)
+					if isNotSupportedFault {
+						log.Infof("observed CnsFault with NotSupported fault cause while attaching volume: %q with vm: %q",
+							volumeID, vm.String())
+
+						// Extract the specific error message from NotSupported fault's FaultMessage array
+						var errorMessages []string
+						for _, faultMsg := range notSupportedFault.FaultMessage {
+							if faultMsg.Message != "" {
+								errorMessages = append(errorMessages, faultMsg.Message)
+							}
+						}
+
+						if len(errorMessages) > 0 {
+							extractedMessage := strings.Join(errorMessages, " - ")
+							log.Infof("NotSupported fault extracted message: %s", extractedMessage)
+							return "", faultType, logger.LogNewErrorf(log,
+								"%q Failed to attach cns volume: %q to node vm: %q. fault: %q. opId: %q",
+								extractedMessage, volumeID, vm.String(), spew.Sdump(volumeOperationRes.Fault), taskInfo.ActivationId)
+						}
+
+						// Fallback to detailed dump for debugging
+						log.Debugf("NotSupported fault details: %+v", spew.Sdump(cnsFault.FaultCause))
+					}
+				}
+			}
+
 			return "", faultType, logger.LogNewErrorf(log, "failed to attach cns volume: %q to node vm: %q. fault: %q. opId: %q",
 				volumeID, vm.String(), spew.Sdump(volumeOperationRes.Fault), taskInfo.ActivationId)
 		}
@@ -2795,20 +2826,29 @@ func (m *defaultManager) createSnapshotWithImprovedIdempotencyCheck(ctx context.
 // This function ensures that no orphaned snapshots are left behind on the vSphere backend
 // in case of failures during the snapshot creation process
 func (m *defaultManager) createSnapshotWithTransaction(ctx context.Context, volumeID string,
-	snapshotID string, extraParams interface{}) (*CnsSnapshotInfo, string, error) {
+	snapshotName string, extraParams interface{}) (*CnsSnapshotInfo, string, error) {
 	log := logger.GetLogger(ctx)
 	var (
 		// Reference to the CreateSnapshot task on CNS.
 		createSnapshotsTask *object.Task
 		// Name of the CnsVolumeOperationRequest instance.
-		instanceName = snapshotID + "-" + volumeID
+		instanceName = snapshotName + "-" + volumeID
 		// Local instance of CreateSnapshot details that needs to be persisted.
 		volumeOperationDetails *cnsvolumeoperationrequest.VolumeOperationRequestDetails
 		// error
 		err                        error
 		quotaInfo                  *cnsvolumeoperationrequest.QuotaDetails
 		isStorageQuotaM2FSSEnabled bool
 	)
+	// By default, external-snapshotter sets the snapshot name prefix to "snapshot-".
+	// This logic will break if the prefix configuration is changed.
+	// In Supervisor deployments, we assume this configuration remains unchanged by admin/DevOps.
+	// In Vanilla deployments, we publish the deployment manifest with the default configuration to ensure consistency.
+	if !strings.HasPrefix(snapshotName, "snapshot-") {
+		return nil, csifault.CSIInternalFault,
+			logger.LogNewErrorf(log, "invalid snapshotName %q: must start with 'snapshot-'", snapshotName)
+	}
+	snapshotID := strings.TrimPrefix(snapshotName, "snapshot-")
 	if extraParams != nil {
 		createSnapParams, ok := extraParams.(*CreateSnapshotExtraParams)
 		if !ok {
@@ -2870,6 +2910,14 @@ func (m *defaultManager) createSnapshotWithTransaction(ctx context.Context, volu
 		faultType := ExtractFaultTypeFromErr(ctx, err)
 		return nil, faultType, logger.LogNewErrorf(log, "failed to create snapshot with error: %v", err)
 	}
+	// Persist the volume operation details.
+	volumeOperationDetails = createRequestDetails(instanceName, volumeID, "", 0, quotaInfo,
+		volumeOperationDetails.OperationDetails.TaskInvocationTimestamp,
+		createSnapshotsTask.Reference().Value, "", "", taskInvocationStatusInProgress, "")
+	if err := m.operationStore.StoreRequestDetails(ctx, volumeOperationDetails); err != nil {
+		// Don't return if CreateSnapshot details can't be stored.
+		log.Warnf("failed to store CreateSnapshot details with error: %v", err)
+	}
 
 	var createSnapshotsTaskInfo *vim25types.TaskInfo
 	var faultType string
@@ -2888,8 +2936,20 @@ func (m *defaultManager) createSnapshotWithTransaction(ctx context.Context, volu
 			"from vCenter %q with err: %v", m.virtualCenter.Config.Host, err)
 	}
 	log.Infof("CreateSnapshots: VolumeID: %q, opId: %q", volumeID, createSnapshotsTaskInfo.ActivationId)
-
-	snapshotCreateResult := interface{}(createSnapshotsTaskInfo).(*cnstypes.CnsSnapshotCreateResult)
+	createSnapshotsTaskResult, err := cns.GetTaskResult(ctx, createSnapshotsTaskInfo)
+	if err != nil || createSnapshotsTaskResult == nil {
+		return nil, "", logger.LogNewErrorf(log, "unable to find the task result for CreateSnapshots task: %q "+
+			"from vCenter %q with err: %v", createSnapshotsTaskInfo.Task.Value, m.virtualCenter.Config.Host, err)
+	}
+	snapshotCreateResult, ok := createSnapshotsTaskResult.(*cnstypes.CnsSnapshotCreateResult)
+	if !ok || snapshotCreateResult == nil {
+		return nil, "", logger.LogNewErrorf(log,
+			"invalid task result: got %T with value %+v", createSnapshotsTaskResult, createSnapshotsTaskResult)
+	}
+	if snapshotCreateResult.Fault != nil {
+		return nil, "", logger.LogNewErrorf(log, "failed to create snapshot %q on volume %q with fault: %+v",
+			instanceName, volumeID, snapshotCreateResult.Fault)
+	}
 	cnsSnapshotInfo := &CnsSnapshotInfo{
 		SnapshotID:                          snapshotCreateResult.Snapshot.SnapshotId.Id,
 		SourceVolumeID:                      snapshotCreateResult.Snapshot.VolumeId.Id,
@@ -2947,14 +3007,15 @@ func (m *defaultManager) CreateSnapshot(
 			}
 		}
 		if createSnapParams != nil && createSnapParams.IsCSITransactionSupportEnabled {
-			var snapcontentPrefix = "snapcontent-"
 			cnssnapshotInfo, fault, err := m.createSnapshotWithTransaction(ctx, volumeID,
-				strings.TrimPrefix(snapshotName, snapcontentPrefix), extraParams)
+				snapshotName, extraParams)
 			if err != nil {
 				if IsNotSupportedFaultType(ctx, fault) {
 					log.Infof("Creating Snapshot with Transaction is not supported. " +
 						"Re-creating Snapshot without setting Snapshot ID in the spec")
 					return m.createSnapshotWithImprovedIdempotencyCheck(ctx, volumeID, snapshotName, extraParams)
+				} else {
+					return nil, logger.LogNewErrorf(log, "failed to create snapshot. error :%+v", err)
 				}
 			}
 			return cnssnapshotInfo, nil