diff --git a/tests/e2e/bootstrap/bootstrap.go b/tests/e2e/bootstrap/bootstrap.go index 3bdcd4a967..96bc59389c 100644 --- a/tests/e2e/bootstrap/bootstrap.go +++ b/tests/e2e/bootstrap/bootstrap.go @@ -26,6 +26,7 @@ import ( cnstypes "github.com/vmware/govmomi/cns/types" "k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e/framework/testfiles" + "sigs.k8s.io/vsphere-csi-driver/v3/tests/e2e/clients/cns" "sigs.k8s.io/vsphere-csi-driver/v3/tests/e2e/clients/vc" "sigs.k8s.io/vsphere-csi-driver/v3/tests/e2e/config" "sigs.k8s.io/vsphere-csi-driver/v3/tests/e2e/constants" @@ -53,6 +54,8 @@ func Bootstrap(optionArgs ...bool) *config.E2eTestConfig { e2eTestConfig.TestInput.Global.VCenterPort, e2eTestConfig.TestInput.Global.User, e2eTestConfig.TestInput.Global.Password) + e2eTestConfig.CnsClient, _ = cns.NewCnsClient(ctx, e2eTestConfig.VcClient.Client) + if framework.TestContext.RepoRoot != "" { testfiles.AddFileSource(testfiles.RootFileSource{Root: framework.TestContext.RepoRoot}) } diff --git a/tests/e2e/config/test_config.go b/tests/e2e/config/test_config.go index c853c2bbf1..3e99eb6adb 100644 --- a/tests/e2e/config/test_config.go +++ b/tests/e2e/config/test_config.go @@ -106,59 +106,60 @@ type TestInputData struct { // This struct has all the testbed related information type TestBedConfig struct { - EsxIp1 string - EsxIp2 string - EsxIp3 string - EsxIp4 string - EsxIp5 string - EsxIp6 string - EsxIp7 string - EsxIp8 string - EsxIp9 string - EsxIp10 string - VcAddress string - VcAddress2 string - VcAddress3 string - MasterIP1 string - MasterIP2 string - MasterIP3 string - IpPortMap map[string]string - MissingEnvVars []string - DefaultlocalhostIP string //= "127.0.0.1" - VcIp2SshPortNum string - VcIp3SshPortNum string - RwxAccessMode bool - Vcptocsi bool - MultipleSvc bool - WindowsEnv bool - Multivc bool - StretchedSVC bool - IsPrivateNetwork bool - K8sMasterIp1PortNum string - K8sMasterIp2PortNum string - K8sMasterIp3PortNum string - VcIp1SshPortNum string - EsxIp1PortNum string - EsxIp2PortNum string - EsxIp3PortNum string - EsxIp4PortNum string - EsxIp5PortNum string - EsxIp6PortNum string - EsxIp7PortNum string - EsxIp8PortNum string - EsxIp9PortNum string - EsxIp10PortNum string - VcVersion string - DefaultCluster *object.ClusterComputeResource - DefaultDatastore *object.Datastore - Name string `default:"worker"` - User string - Location string - VcIp string - VcVmName string - EsxHosts []map[string]string - Podname string - Datastores []map[string]string + EsxIp1 string + EsxIp2 string + EsxIp3 string + EsxIp4 string + EsxIp5 string + EsxIp6 string + EsxIp7 string + EsxIp8 string + EsxIp9 string + EsxIp10 string + VcAddress string + VcAddress2 string + VcAddress3 string + MasterIP1 string + MasterIP2 string + MasterIP3 string + IpPortMap map[string]string + MissingEnvVars []string + DefaultlocalhostIP string //= "127.0.0.1" + VcIp2SshPortNum string + VcIp3SshPortNum string + RwxAccessMode bool + Vcptocsi bool + MultipleSvc bool + WindowsEnv bool + Multivc bool + StretchedSVC bool + IsPrivateNetwork bool + K8sMasterIp1PortNum string + K8sMasterIp2PortNum string + K8sMasterIp3PortNum string + VcIp1SshPortNum string + EsxIp1PortNum string + EsxIp2PortNum string + EsxIp3PortNum string + EsxIp4PortNum string + EsxIp5PortNum string + EsxIp6PortNum string + EsxIp7PortNum string + EsxIp8PortNum string + EsxIp9PortNum string + EsxIp10PortNum string + VcVersion string + DefaultCluster *object.ClusterComputeResource + DefaultDatastore *object.Datastore + Name string `default:"worker"` + User string + Location string + VcIp string + VcVmName string + EsxHosts []map[string]string + Podname string + Datastores []map[string]string + WcpVsanDirectCluster bool } // NetPermissionConfig consists of information used to restrict the diff --git a/tests/e2e/constants/env_constants.go b/tests/e2e/constants/env_constants.go index 822d586fd0..6539adc010 100644 --- a/tests/e2e/constants/env_constants.go +++ b/tests/e2e/constants/env_constants.go @@ -225,3 +225,9 @@ const ( EnvIsDevopsUser = "IS_DEVOPS_USER" EnvDevopsUserName = "DEVOPS_USERNAME" ) + +// For creating disk on target datastore +var ( + EnvStoragePolicy = "STORAGE_POLICY_NAME" + EnvRestoreStoragePolicy = "RESTORE_STORAGE_POLICY" +) diff --git a/tests/e2e/csisnapshot/util.go b/tests/e2e/csisnapshot/util.go index 7d5ae0343e..4fdf6db7cd 100644 --- a/tests/e2e/csisnapshot/util.go +++ b/tests/e2e/csisnapshot/util.go @@ -27,15 +27,21 @@ import ( "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" v1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" "k8s.io/kubernetes/test/e2e/framework" + e2ekubectl "k8s.io/kubernetes/test/e2e/framework/kubectl" + fpv "k8s.io/kubernetes/test/e2e/framework/pv" "sigs.k8s.io/vsphere-csi-driver/v3/tests/e2e/config" "sigs.k8s.io/vsphere-csi-driver/v3/tests/e2e/constants" "sigs.k8s.io/vsphere-csi-driver/v3/tests/e2e/env" + "sigs.k8s.io/vsphere-csi-driver/v3/tests/e2e/k8testutil" "sigs.k8s.io/vsphere-csi-driver/v3/tests/e2e/vcutil" ) @@ -389,3 +395,177 @@ func WaitForVolumeSnapshotContentToBeDeleted(client snapclient.Clientset, ctx co }) return waitErr } + +/* +Create volume snapshot +*/ +func CreateVolumeSnapshot(ctx context.Context, e2eTestConfig *config.E2eTestConfig, namespace string, pvclaim *v1.PersistentVolumeClaim, pv []*v1.PersistentVolume, diskSize string) (*snapV1.VolumeSnapshot, *snapV1.VolumeSnapshotContent) { + // Create or get volume snapshot class + ginkgo.By("Get or create volume snapshot class") + snapc := GetSnashotClientSet(e2eTestConfig) + volumeSnapshotClass, err := CreateVolumeSnapshotClass(ctx, e2eTestConfig, snapc, constants.DeletionPolicy) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + // Create volume snapshot + ginkgo.By("Create a volume snapshot") + performCnsQueryVolumeSnapshot := false + if e2eTestConfig.TestInput.ClusterFlavor.SupervisorCluster { + performCnsQueryVolumeSnapshot = true + } + volumeSnapshot, snapshotContent, _, + _, _, _, err := CreateDynamicVolumeSnapshot(ctx, e2eTestConfig, namespace, snapc, volumeSnapshotClass, + pvclaim, pv[0].Spec.CSI.VolumeHandle, diskSize, performCnsQueryVolumeSnapshot) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + return volumeSnapshot, snapshotContent +} + +/* +Get snashot client set +*/ +func GetSnashotClientSet(e2eTestConfig *config.E2eTestConfig) *snapclient.Clientset { + var restConfig *rest.Config + if e2eTestConfig.TestInput.ClusterFlavor.GuestCluster { + restConfig = GetRestConfigClientForGuestCluster(nil) + } else { + restConfig = vcutil.GetRestConfigClient(e2eTestConfig) + } + snapc, err := snapclient.NewForConfig(restConfig) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + return snapc +} + +func GetRestConfigClientForGuestCluster(guestClusterRestConfig *rest.Config) *rest.Config { + var err error + if guestClusterRestConfig == nil { + if k8senv := env.GetAndExpectStringEnvVar("KUBECONFIG"); k8senv != "" { + guestClusterRestConfig, err = clientcmd.BuildConfigFromFlags("", k8senv) + } + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + } + return guestClusterRestConfig +} + +// verifyVolumeRestoreOperation verifies if volume(PVC) restore from given snapshot +// and creates pod and checks attach volume operation if verifyPodCreation is set to true +func VerifyVolumeRestoreOperation(ctx context.Context, vs *config.E2eTestConfig, client kubernetes.Interface, + namespace string, storageclass *storagev1.StorageClass, + volumeSnapshot *snapV1.VolumeSnapshot, diskSize string, + verifyPodCreation bool) (*v1.PersistentVolumeClaim, []*v1.PersistentVolume, *v1.Pod) { + + ginkgo.By("Create PVC from snapshot") + pvcSpec := GetPersistentVolumeClaimSpecWithDatasource(namespace, diskSize, storageclass, nil, + v1.ReadWriteOnce, volumeSnapshot.Name, constants.Snapshotapigroup) + + pvclaim2, err := fpv.CreatePVC(ctx, client, namespace, pvcSpec) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + persistentvolumes2, err := fpv.WaitForPVClaimBoundPhase(ctx, client, + []*v1.PersistentVolumeClaim{pvclaim2}, framework.ClaimProvisionTimeout) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + volHandle2 := persistentvolumes2[0].Spec.CSI.VolumeHandle + if vs.TestInput.ClusterFlavor.GuestCluster { + volHandle2 = k8testutil.GetVolumeIDFromSupervisorCluster(volHandle2) + } + gomega.Expect(volHandle2).NotTo(gomega.BeEmpty()) + + var pod *v1.Pod + if verifyPodCreation { + // Create a Pod to use this PVC, and verify volume has been attached + ginkgo.By("Creating pod to attach PV to the node") + pod, err = k8testutil.CreatePod(ctx, vs, client, namespace, nil, + []*v1.PersistentVolumeClaim{pvclaim2}, false, constants.ExecRWXCommandPod1) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + var vmUUID string + var exists bool + nodeName := pod.Spec.NodeName + + if vs.TestInput.ClusterFlavor.VanillaCluster { + vmUUID = k8testutil.GetNodeUUID(ctx, client, pod.Spec.NodeName) + } else if vs.TestInput.ClusterFlavor.GuestCluster { + vmUUID, err = vcutil.GetVMUUIDFromNodeName(vs, pod.Spec.NodeName) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + } else if vs.TestInput.ClusterFlavor.SupervisorCluster { + annotations := pod.Annotations + vmUUID, exists = annotations[constants.VmUUIDLabel] + gomega.Expect(exists).To(gomega.BeTrue(), fmt.Sprintf("Pod doesn't have %s annotation", constants.VmUUIDLabel)) + _, err := vcutil.GetVMByUUID(ctx, vs, vmUUID) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + } + + ginkgo.By(fmt.Sprintf("Verify volume: %s is attached to the node: %s", volHandle2, nodeName)) + isDiskAttached, err := vcutil.IsVolumeAttachedToVM(client, vs, volHandle2, vmUUID) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(isDiskAttached).To(gomega.BeTrue(), "Volume is not attached to the node") + + ginkgo.By("Verify the volume is accessible and Read/write is possible") + var cmd []string + if vs.TestInput.TestBedInfo.WindowsEnv { + cmd = []string{"exec", pod.Name, "--namespace=" + namespace, "powershell.exe", "cat ", constants.FilePathPod1} + } else { + cmd = []string{"exec", pod.Name, "--namespace=" + namespace, "--", "/bin/sh", "-c", + "cat ", constants.FilePathPod1} + } + _ = e2ekubectl.RunKubectlOrDie(namespace, cmd...) + //gomega.Expect(strings.Contains(output, "Hello message from Pod1")).NotTo(gomega.BeFalse()) + + var wrtiecmd []string + if vs.TestInput.TestBedInfo.WindowsEnv { + wrtiecmd = []string{"exec", pod.Name, "--namespace=" + namespace, "powershell.exe", + "Add-Content /mnt/volume1/Pod1.html 'Hello message from test into Pod1'"} + } else { + wrtiecmd = []string{"exec", pod.Name, "--namespace=" + namespace, "--", "/bin/sh", "-c", + "echo 'Hello message from test into Pod1' >> /mnt/volume1/Pod1.html"} + } + e2ekubectl.RunKubectlOrDie(namespace, wrtiecmd...) + _ = e2ekubectl.RunKubectlOrDie(namespace, cmd...) + //gomega.Expect(strings.Contains(output, "Hello message from test into Pod1")).NotTo(gomega.BeFalse()) + return pvclaim2, persistentvolumes2, pod + } + return pvclaim2, persistentvolumes2, pod +} + +// getPersistentVolumeClaimSpecWithDatasource return the PersistentVolumeClaim +// spec with specified storage class. +func GetPersistentVolumeClaimSpecWithDatasource(namespace string, ds string, storageclass *storagev1.StorageClass, + pvclaimlabels map[string]string, accessMode v1.PersistentVolumeAccessMode, + datasourceName string, snapshotapigroup string) *v1.PersistentVolumeClaim { + disksize := constants.DiskSize + if ds != "" { + disksize = ds + } + if accessMode == "" { + // If accessMode is not specified, set the default accessMode. + accessMode = v1.ReadWriteOnce + } + claim := &v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "pvc-", + Namespace: namespace, + }, + Spec: v1.PersistentVolumeClaimSpec{ + AccessModes: []v1.PersistentVolumeAccessMode{ + accessMode, + }, + Resources: v1.VolumeResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceName(v1.ResourceStorage): resource.MustParse(disksize), + }, + }, + StorageClassName: &(storageclass.Name), + DataSource: &v1.TypedLocalObjectReference{ + APIGroup: &snapshotapigroup, + Kind: "VolumeSnapshot", + Name: datasourceName, + }, + }, + } + + if pvclaimlabels != nil { + claim.Labels = pvclaimlabels + } + + return claim +} diff --git a/tests/e2e/k8testutil/util.go b/tests/e2e/k8testutil/util.go index 7e2845328c..9811a1f6ca 100644 --- a/tests/e2e/k8testutil/util.go +++ b/tests/e2e/k8testutil/util.go @@ -32,6 +32,7 @@ import ( "os/exec" "path/filepath" "reflect" + "regexp" "sort" "strconv" "strings" @@ -94,6 +95,7 @@ import ( "sigs.k8s.io/vsphere-csi-driver/v3/tests/e2e/config" "sigs.k8s.io/vsphere-csi-driver/v3/tests/e2e/constants" "sigs.k8s.io/vsphere-csi-driver/v3/tests/e2e/env" + "sigs.k8s.io/vsphere-csi-driver/v3/tests/e2e/vcutil" authenticationv1 "k8s.io/api/authentication/v1" @@ -2540,7 +2542,7 @@ func CreateEmptyFilesOnVSphereVolume(e2eTestConfig *config.E2eTestConfig, func CreateService(ns string, c clientset.Interface) *v1.Service { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - svcManifestFilePath := filepath.Join(constants.ManifestPath, "service.yaml") + svcManifestFilePath := filepath.Join("..", constants.ManifestPath, "service.yaml") framework.Logf("Parsing service from %v", svcManifestFilePath) svc, err := manifest.SvcFromManifest(svcManifestFilePath) framework.ExpectNoError(err) @@ -2561,7 +2563,7 @@ func DeleteService(ns string, c clientset.Interface, service *v1.Service) { // GetStatefulSetFromManifest creates a StatefulSet from the statefulset.yaml // file present in the manifest path. func GetStatefulSetFromManifest(e2eTestConfig *config.TestInputData, ns string) *appsv1.StatefulSet { - ssManifestFilePath := filepath.Join(constants.ManifestPath, "statefulset.yaml") + ssManifestFilePath := filepath.Join("..", constants.ManifestPath, "statefulset.yaml") framework.Logf("Parsing statefulset from %v", ssManifestFilePath) ss, err := manifest.StatefulSetFromManifest(ssManifestFilePath, ns) framework.ExpectNoError(err) @@ -7720,3 +7722,626 @@ func CreateStatefulSet(ns string, ss *appsv1.StatefulSet, c clientset.Interface) framework.ExpectNoError(err) fss.WaitForRunningAndReady(ctx, c, *ss.Spec.Replicas, ss) } + +/* +This function creates a wcp namespace in a vSphere supervisor Cluster, associating it +with multiple storage policies and zones. +It constructs an API request and sends it to the vSphere REST API. +*/ +func CreatetWcpNsWithZonesAndPolicies(e2eTestConfig *config.E2eTestConfig, + vcRestSessionId string, storagePolicyId []string, + supervisorId string, zoneNames []string, + vmClass string, contentLibId string) (string, int, error) { + + r := rand.New(rand.NewSource(time.Now().Unix())) + namespace := fmt.Sprintf("csi-%v", r.Intn(10000)) + initailUrl := CreateInitialNsApiCallUrl(e2eTestConfig) + nsCreationUrl := initailUrl + "v2" + + var storageSpecs []map[string]string + for _, policyId := range storagePolicyId { + storageSpecs = append(storageSpecs, map[string]string{"policy": policyId}) + } + + var zones []map[string]string + for _, zone := range zoneNames { + zones = append(zones, map[string]string{"name": zone}) + } + + // Create request body struct + requestBody := map[string]interface{}{ + "namespace": namespace, + "storage_specs": storageSpecs, + "supervisor": supervisorId, + "zones": zones, + } + + // Add vm_service_spec only if vmClass and contentLibId are provided + if vmClass != "" && contentLibId != "" { + requestBody["vm_service_spec"] = map[string]interface{}{ + "vm_classes": []string{vmClass}, + "content_libraries": []string{contentLibId}, + } + } + + reqBodyBytes, err := json.Marshal(requestBody) + if err != nil { + return "", 500, fmt.Errorf("error marshalling request body: %w", err) + } + + reqBody := string(reqBodyBytes) + fmt.Println(reqBody) + + // Make the API request + _, statusCode := InvokeVCRestAPIPostRequest(vcRestSessionId, nsCreationUrl, reqBody) + + return namespace, statusCode, nil +} + +// invokeVCRestAPIPostRequest invokes POST on given VC REST URL using the passed session token and request body +func InvokeVCRestAPIPostRequest(vcRestSessionId string, url string, reqBody string) ([]byte, int) { + transCfg := &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + httpClient := &http.Client{Transport: transCfg} + framework.Logf("Invoking POST on url: %s", url) + req, err := http.NewRequest("POST", url, strings.NewReader(reqBody)) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + req.Header.Add(constants.VcRestSessionIdHeaderName, vcRestSessionId) + req.Header.Add("Content-type", "application/json") + + resp, statusCode := HttpRequest(httpClient, req) + + return resp, statusCode +} + +/* +This util will create initial namespace get/post api call request +*/ +func CreateInitialNsApiCallUrl(e2eTestConfig *config.E2eTestConfig) string { + vcIp := e2eTestConfig.TestInput.Global.VCenterHostname + + isPrivateNetwork := env.GetBoolEnvVarOrDefault("IS_PRIVATE_NETWORK", false) + if isPrivateNetwork { + vcIp = env.GetStringEnvVarOrDefault("LOCAL_HOST_IP", constants.DefaultlocalhostIP) + } + + initialUrl := "https://" + vcIp + ":" + e2eTestConfig.TestInput.Global.VCenterPort + + "/api/vcenter/namespaces/instances/" + + return initialUrl +} + +// getSvcId fetches the ID of the Supervisor cluster +func GetSvcId(vcRestSessionId string, e2eTestConfig *config.E2eTestConfig) string { + + isPrivateNetwork := env.GetBoolEnvVarOrDefault("IS_PRIVATE_NETWORK", false) + vCenterIp := e2eTestConfig.TestInput.Global.VCenterHostname + if isPrivateNetwork { + vCenterIp = env.GetStringEnvVarOrDefault("LOCAL_HOST_IP", constants.DefaultlocalhostIP) + } + + svcIdFetchUrl := "https://" + vCenterIp + ":" + e2eTestConfig.TestInput.Global.VCenterPort + + "/api/vcenter/namespace-management/supervisors/summaries" + + resp, statusCode := InvokeVCRestAPIGetRequest(vcRestSessionId, svcIdFetchUrl) + gomega.Expect(statusCode).Should(gomega.BeNumerically("==", 200)) + + var v map[string]interface{} + gomega.Expect(json.Unmarshal(resp, &v)).NotTo(gomega.HaveOccurred()) + framework.Logf("Supervisor summary: %v", v) + return v["items"].([]interface{})[0].(map[string]interface{})["supervisor"].(string) +} + +// invokeVCRestAPIGetRequest invokes GET on given VC REST URL using the passed session token and verifies that the +// return status code is 200 +func InvokeVCRestAPIGetRequest(vcRestSessionId string, url string) ([]byte, int) { + transCfg := &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + httpClient := &http.Client{Transport: transCfg} + framework.Logf("Invoking GET on url: %s", url) + req, err := http.NewRequest("GET", url, nil) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + req.Header.Add(constants.VcRestSessionIdHeaderName, vcRestSessionId) + + resp, statusCode := HttpRequest(httpClient, req) + + return resp, statusCode +} + +// delTestWcpNs triggeres a wcp namespace deletion asynchronously +func DelTestWcpNs(e2eTestConfig *config.E2eTestConfig, vcRestSessionId string, namespace string) { + vcIp := e2eTestConfig.TestInput.Global.VCenterHostname + isPrivateNetwork := env.GetBoolEnvVarOrDefault("IS_PRIVATE_NETWORK", false) + if isPrivateNetwork { + vcIp = env.GetStringEnvVarOrDefault("LOCAL_HOST_IP", constants.DefaultlocalhostIP) + } + nsDeletionUrl := "https://" + vcIp + ":" + e2eTestConfig.TestInput.Global.VCenterPort + + "/api/vcenter/namespaces/instances/" + namespace + _, statusCode := InvokeVCRestAPIDeleteRequest(vcRestSessionId, nsDeletionUrl) + gomega.Expect(statusCode).Should(gomega.BeNumerically("==", 204)) + framework.Logf("Successfully Deleted namepsace %v in SVC.", namespace) +} + +// invokeVCRestAPIDeleteRequest invokes DELETE on given VC REST URL using the passed session token +func InvokeVCRestAPIDeleteRequest(vcRestSessionId string, url string) ([]byte, int) { + transCfg := &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + httpClient := &http.Client{Transport: transCfg} + framework.Logf("Invoking DELETE on url: %s", url) + req, err := http.NewRequest("DELETE", url, nil) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + req.Header.Add(constants.VcRestSessionIdHeaderName, vcRestSessionId) + + resp, statusCode := HttpRequest(httpClient, req) + + return resp, statusCode +} + +// Create N number of PVC and attach them to Pod +func CreateMultiplePvcPod(ctx context.Context, e2eTestConfig *config.E2eTestConfig, client clientset.Interface, namespace string, storageclass *storagev1.StorageClass, numOfPVc int, doCreatePod bool, doCreateDep bool) map[*v1.PersistentVolumeClaim][]*v1.PersistentVolume { + + volumeMap := map[*v1.PersistentVolumeClaim][]*v1.PersistentVolume{} + + // Create PVC and verify PVC is bound + for i := 0; i < numOfPVc; i++ { + ginkgo.By(fmt.Sprintf("Creating PVC in iteration: %v", + i)) + pvclaim, pv := CreateAndValidatePvc(ctx, client, namespace, storageclass) + volumeMap[pvclaim] = pv + + // Create Pod and attach to PVC + if doCreatePod || doCreateDep { + CreatePodForPvc(ctx, e2eTestConfig, client, namespace, []*v1.PersistentVolumeClaim{pvclaim}, doCreatePod, doCreateDep) + } + } + + return volumeMap +} + +func CreatePodForPvc(ctx context.Context, e2eTestConfig *config.E2eTestConfig, client clientset.Interface, namespace string, pvclaim []*v1.PersistentVolumeClaim, deCreatePod bool, doCreateDep bool) (*v1.Pod, *appsv1.Deployment) { + ginkgo.By("Create Pod to attach to Pvc") + var pod *v1.Pod + var dep *appsv1.Deployment + var err error + if deCreatePod { + pod, err = CreatePod(ctx, e2eTestConfig, client, namespace, nil, pvclaim, false, + constants.ExecRWXCommandPod1) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + } else if doCreateDep { + labelsMap := make(map[string]string) + labelsMap["app"] = "test" + dep, err = CreateDeployment(ctx, e2eTestConfig, client, 1, labelsMap, nil, namespace, + pvclaim, constants.ExecRWXCommandPod1, false, constants.NginxImage) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + } + return pod, dep +} + +/* +Create and validate PVC status +*/ +func CreateAndValidatePvc(ctx context.Context, client clientset.Interface, namespace string, storageclass *storagev1.StorageClass) (*v1.PersistentVolumeClaim, []*v1.PersistentVolume) { + ginkgo.By("Create PVC") + pvclaim, err := CreatePVC(ctx, client, namespace, nil, "", storageclass, "") + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + // Validate PVC is bound + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + pv, err := fpv.WaitForPVClaimBoundPhase(ctx, + client, []*v1.PersistentVolumeClaim{pvclaim}, framework.ClaimProvisionTimeout) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + return pvclaim, pv +} + +// onlineVolumeResizeCheck this method increases the PVC size, which is attached +// to POD. +func OnlineVolumeResizeCheck(vs *config.E2eTestConfig, f *framework.Framework, client clientset.Interface, + namespace string, svcPVCName string, volHandle string, pvclaim *v1.PersistentVolumeClaim, pod *v1.Pod) { + var originalSizeInMb int64 + var err error + // Fetch original FileSystemSize. + ginkgo.By("Verify filesystem size for mount point /mnt/volume1 before expansion") + originalSizeInMb, err = GetFileSystemSizeForOsType(vs, f, client, pod) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + // Resize PVC. + // Modify PVC spec to trigger volume expansion. + ginkgo.By("Expanding current pvc") + currentPvcSize := pvclaim.Spec.Resources.Requests[v1.ResourceStorage] + newSize := currentPvcSize.DeepCopy() + newSize.Add(resource.MustParse("1Gi")) + framework.Logf("currentPvcSize %v, newSize %v", currentPvcSize, newSize) + pvclaim, err = ExpandPVCSize(pvclaim, newSize, client) + gomega.Expect(pvclaim).NotTo(gomega.BeNil()) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + _, err = WaitForFSResizeInSvc(svcPVCName) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + ginkgo.By("Waiting for file system resize to finish") + pvclaim, err = WaitForFSResize(pvclaim, client) + framework.ExpectNoError(err, "while waiting for fs resize to finish") + + pvcConditions := pvclaim.Status.Conditions + ExpectEqual(len(pvcConditions), 0, "pvc should not have conditions") + + ginkgo.By(fmt.Sprintf("Invoking QueryCNSVolumeWithResult with VolumeID: %s", volHandle)) + queryResult, err := vcutil.QueryCNSVolumeWithResult(vs, volHandle) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + if len(queryResult.Volumes) == 0 { + err = fmt.Errorf("QueryCNSVolumeWithResult returned no volume") + } + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + var fsSize int64 + ginkgo.By("Verify filesystem size for mount point /mnt/volume1") + fsSize, err = GetFileSystemSizeForOsType(vs, f, client, pod) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + framework.Logf("File system size after expansion : %d", fsSize) + // Filesystem size may be smaller than the size of the block volume + // so here we are checking if the new filesystem size is greater than + // the original volume size as the filesystem is formatted for the + // first time. + gomega.Expect(fsSize).Should(gomega.BeNumerically(">", originalSizeInMb), + fmt.Sprintf("error updating filesystem size for %q. Resulting filesystem size is %d", pvclaim.Name, fsSize)) + ginkgo.By("File system resize finished successfully") + + framework.Logf("Online volume expansion in GC PVC is successful") + +} + +// getFileSystemSizeForOsType returns the file system size of the volume in respective OS type +func GetFileSystemSizeForOsType(vs *config.E2eTestConfig, f *framework.Framework, client clientset.Interface, pod *v1.Pod) (int64, error) { + var fsSize int64 + var err error + if vs.TestInput.TestBedInfo.WindowsEnv { + fsSize, err = GetWindowsFileSystemSize(client, vs, pod) + } else { + fsSize, err = getFSSizeMb(vs, f, pod) + } + return fsSize, err +} + +// getFSSizeMb returns filesystem size in Mb +func getFSSizeMb(vs *config.E2eTestConfig, f *framework.Framework, pod *v1.Pod) (int64, error) { + var output string + var err error + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + if vs.TestInput.ClusterFlavor.SupervisorCluster { + namespace := vcutil.GetNamespaceToRunTests(f, vs) + var cmd []string + if vs.TestInput.TestBedInfo.WcpVsanDirectCluster { + cmd = []string{"exec", pod.Name, "--namespace=" + namespace, "--", "/bin/sh", "-c", "df -Tkm | grep /data0"} + } else { + cmd = []string{"exec", pod.Name, "--namespace=" + namespace, "--", "/bin/sh", "-c", "df -Tkm | grep /mnt/volume1"} + } + output = e2ekubectl.RunKubectlOrDie(namespace, cmd...) + gomega.Expect(strings.Contains(output, constants.Ext4FSType)).NotTo(gomega.BeFalse()) + } else { + output, _, err = fpod.ExecShellInPodWithFullOutput(ctx, f, pod.Name, "df -T -m | grep /mnt/volume1") + if err != nil { + return -1, fmt.Errorf("unable to find mount path via `df -T`: %v", err) + } + } + + arrMountOut := strings.Fields(string(output)) + if len(arrMountOut) <= 0 { + return -1, fmt.Errorf("error when parsing output of `df -T`. output: %s", string(output)) + } + var devicePath, strSize string + devicePath = arrMountOut[0] + if devicePath == "" { + return -1, fmt.Errorf("error when parsing output of `df -T` to find out devicePath of /mnt/volume1. output: %s", + string(output)) + } + strSize = arrMountOut[2] + if strSize == "" { + return -1, fmt.Errorf("error when parsing output of `df -T` to find out size of /mnt/volume1: output: %s", + string(output)) + } + + intSizeInMb, err := strconv.ParseInt(strSize, 10, 64) + if err != nil { + return -1, fmt.Errorf("failed to parse size %s into int size", strSize) + } + + return intSizeInMb, nil +} + +// waitForFSResize waits for the filesystem in the pv to be resized. +func WaitForFSResizeInSvc(svcPVCName string) (*v1.PersistentVolumeClaim, error) { + svcClient, _ := GetSvcClientAndNamespace() + svcPvc := GetPVCFromSupervisorCluster(svcPVCName) + return WaitForFSResize(svcPvc, svcClient) +} + +// waitForFSResize waits for the filesystem in the pv to be resized +func WaitForFSResize(pvc *v1.PersistentVolumeClaim, c clientset.Interface) (*v1.PersistentVolumeClaim, error) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + var updatedPVC *v1.PersistentVolumeClaim + waitErr := wait.PollUntilContextTimeout(ctx, constants.ResizePollInterval, constants.TotalResizeWaitPeriod, true, + func(ctx context.Context) (bool, error) { + var err error + updatedPVC, err = c.CoreV1().PersistentVolumeClaims(pvc.Namespace).Get(ctx, pvc.Name, metav1.GetOptions{}) + + if err != nil { + return false, fmt.Errorf("error fetching pvc %q for checking for resize status : %v", pvc.Name, err) + } + + pvcSize := updatedPVC.Spec.Resources.Requests[v1.ResourceStorage] + pvcStatusSize := updatedPVC.Status.Capacity[v1.ResourceStorage] + + // If pvc's status field size is greater than or equal to pvc's size then done + if pvcStatusSize.Cmp(pvcSize) >= 0 { + return true, nil + } + return false, nil + }) + return updatedPVC, waitErr +} + +// expectEqual expects the specified two are the same, otherwise an exception raises +func ExpectEqual(actual interface{}, extra interface{}, explain ...interface{}) { + gomega.ExpectWithOffset(1, actual).To(gomega.Equal(extra), explain...) +} + +// verifyOfflineVolumeExpansionOnGc is a util method which helps in verifying offline volume expansion on gc +func VerifyOfflineVolumeExpansionOnGc(vs *config.E2eTestConfig, ctx context.Context, client clientset.Interface, + pvclaim *v1.PersistentVolumeClaim, svcPVCName string, namespace string, volHandle string, + pod *v1.Pod, pv *v1.PersistentVolume, f *framework.Framework) { + ginkgo.By("Check filesystem size for mount point /mnt/volume1 before expansion") + originalFsSize, err := GetFileSystemSizeForOsType(vs, f, client, pod) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + // Delete POD. + ginkgo.By(fmt.Sprintf("Deleting the pod %s in namespace %s before expansion", pod.Name, namespace)) + err = fpod.DeletePodWithWait(ctx, client, pod) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + ginkgo.By("Verify volume is detached from the node before expansion") + isDiskDetached, err := WaitForVolumeDetachedFromNode(client, + pv.Spec.CSI.VolumeHandle, pod.Spec.NodeName, vs) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(isDiskDetached).To(gomega.BeTrue(), + fmt.Sprintf("Volume %q is not detached from the node %q", volHandle, pod.Spec.NodeName)) + + // Modify PVC spec to trigger volume expansion. We expand the PVC while + // no pod is using it to ensure offline expansion. + ginkgo.By("Expanding current pvc") + currentPvcSize := pvclaim.Spec.Resources.Requests[v1.ResourceStorage] + newSize := currentPvcSize.DeepCopy() + newSize.Add(resource.MustParse(constants.DiskSize)) + framework.Logf("currentPvcSize %v, newSize %v", currentPvcSize, newSize) + pvclaim, err = ExpandPVCSize(pvclaim, newSize, client) + framework.ExpectNoError(err, "While updating pvc for more size") + gomega.Expect(pvclaim).NotTo(gomega.BeNil()) + + pvcSize := pvclaim.Spec.Resources.Requests[v1.ResourceStorage] + if pvcSize.Cmp(newSize) != 0 { + framework.Failf("error updating pvc size %q", pvclaim.Name) + } + ginkgo.By("Checking for PVC request size change on SVC PVC") + b, err := VerifyPvcRequestedSizeUpdateInSupervisorWithWait(svcPVCName, newSize) + gomega.Expect(b).To(gomega.BeTrue()) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + ginkgo.By("Waiting for controller volume resize to finish") + err = WaitForPvResizeForGivenPvc(pvclaim, client, vs, constants.TotalResizeWaitPeriod) + framework.ExpectNoError(err, "While waiting for pvc resize to finish") + + ginkgo.By("Checking for resize on SVC PV") + VerifyPVSizeinSupervisor(svcPVCName, newSize) + + ginkgo.By("Checking for 'FileSystemResizePending' status condition on SVC PVC") + err = WaitForSvcPvcToReachFileSystemResizePendingCondition(ctx, svcPVCName, constants.PollTimeout) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + ginkgo.By("Checking for conditions on pvc") + pvclaim, err = WaitForPVCToReachFileSystemResizePendingCondition(client, namespace, pvclaim.Name, constants.PollTimeout) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + ginkgo.By(fmt.Sprintf("Invoking QueryCNSVolumeWithResult with VolumeID: %s", volHandle)) + queryResult, err := vcutil.QueryCNSVolumeWithResult(vs, volHandle) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + if len(queryResult.Volumes) == 0 { + err = fmt.Errorf("QueryCNSVolumeWithResult returned no volume") + } + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + ginkgo.By("Verifying disk size requested in volume expansion is honored") + newSizeInMb := ConvertGiStrToMibInt64(newSize) + if queryResult.Volumes[0].BackingObjectDetails.(*cnstypes.CnsBlockBackingDetails).CapacityInMb != newSizeInMb { + err = fmt.Errorf("got wrong disk size after volume expansion") + } + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + // Create a new Pod to use this PVC, and verify volume has been attached. + ginkgo.By("Creating a new pod to attach PV again to the node") + pod, err = CreatePod(ctx, vs, client, namespace, nil, []*v1.PersistentVolumeClaim{pvclaim}, false, constants.ExecCommand) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + defer func() { + ginkgo.By("Delete pod") + err = fpod.DeletePodWithWait(ctx, client, pod) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }() + + ginkgo.By(fmt.Sprintf("Verify volume after expansion: %s is attached to the node: %s", + pv.Spec.CSI.VolumeHandle, pod.Spec.NodeName)) + vmUUID, err := vcutil.GetVMUUIDFromNodeName(vs, pod.Spec.NodeName) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + isDiskAttached, err := vcutil.IsVolumeAttachedToVM(client, vs, volHandle, vmUUID) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(isDiskAttached).To(gomega.BeTrue(), "Volume is not attached to the node") + + ginkgo.By("Waiting for file system resize to finish") + pvclaim, err = WaitForFSResize(pvclaim, client) + framework.ExpectNoError(err, "while waiting for fs resize to finish") + + pvcConditions := pvclaim.Status.Conditions + ExpectEqual(len(pvcConditions), 0, "pvc should not have conditions") + + ginkgo.By("Verify filesystem size for mount point /mnt/volume1 after expansion") + fsSize, err := GetFileSystemSizeForOsType(vs, f, client, pod) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + // Filesystem size may be smaller than the size of the block volume. + // Here since filesystem was already formatted on the original volume, + // we can compare the new filesystem size with the original filesystem size. + if fsSize < originalFsSize { + framework.Failf("error updating filesystem size for %q. Resulting filesystem size is %d", pvclaim.Name, fsSize) + } + + ginkgo.By("File system resize finished successfully in GC") + ginkgo.By("Checking for PVC resize completion on SVC PVC") + _, err = WaitForFSResizeInSvc(svcPVCName) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + +} + +// convertGiStrToMibInt64 returns integer numbers of Mb equivalent to string +// of the form \d+Gi. +func ConvertGiStrToMibInt64(size resource.Quantity) int64 { + r, err := regexp.Compile("[0-9]+") + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + sizeInt, err := strconv.Atoi(r.FindString(size.String())) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + return int64(sizeInt * 1024) +} + +// verifyPvcRequestedSizeUpdateInSupervisorWithWait waits till +// Pvc.Spec.Resources.Requests[v1.ResourceStorage] matches with given size +// in SVC. +func VerifyPvcRequestedSizeUpdateInSupervisorWithWait(pvcName string, size resource.Quantity) (bool, error) { + var b bool + waitErr := wait.PollUntilContextTimeout(context.Background(), constants.ResizePollInterval, constants.TotalResizeWaitPeriod, true, + func(ctx context.Context) (bool, error) { + b = VerifyPVCRequestedSizeInSupervisor(pvcName, size) + return b, nil + }) + return b, waitErr +} + +// verifyPVCRequestedSizeInSupervisor compares +// Pvc.Spec.Resources.Requests[v1.ResourceStorage] with given size in SVC. +func VerifyPVCRequestedSizeInSupervisor(pvcName string, size resource.Quantity) bool { + SvcPvc := GetPVCFromSupervisorCluster(pvcName) + pvcSize := SvcPvc.Spec.Resources.Requests[v1.ResourceStorage] + framework.Logf("SVC PVC requested size: %v", pvcSize) + return pvcSize.Cmp(size) == 0 +} + +// verifyPVSizeinSupervisor compares PV.Spec.Capacity[v1.ResourceStorage] in +// SVC with the size passed here. +func VerifyPVSizeinSupervisor(svcPVCName string, newSize resource.Quantity) { + svcPV := GetPvFromSupervisorCluster(svcPVCName) + svcPVSize := svcPV.Spec.Capacity[v1.ResourceStorage] + // If pv size is greater or equal to requested size that means controller + // resize is finished. + gomega.Expect(svcPVSize.Cmp(newSize) >= 0).To(gomega.BeTrue()) +} + +// waitForSvcPvcToReachFileSystemResizePendingCondition waits for SVC PVC to +// reach FileSystemResizePendingCondition status condition. +func WaitForSvcPvcToReachFileSystemResizePendingCondition(ctx context.Context, svcPvcName string, + timeout time.Duration) error { + waitErr := wait.PollUntilContextTimeout(ctx, constants.ResizePollInterval, timeout, true, + func(ctx context.Context) (bool, error) { + _, err := CheckSvcPvcHasGivenStatusCondition(svcPvcName, true, v1.PersistentVolumeClaimFileSystemResizePending) + if err == nil { + return true, nil + } + if strings.Contains(err.Error(), "not matching") { + return false, nil + } + return false, err + }) + return waitErr +} + +// checkSvcPvcHasGivenStatusCondition checks if the status condition in SVC PVC +// matches with the one we want. +func CheckSvcPvcHasGivenStatusCondition(pvcName string, conditionsPresent bool, + condition v1.PersistentVolumeClaimConditionType) (*v1.PersistentVolumeClaim, error) { + svcClient, svcNamespace := GetSvcClientAndNamespace() + return CheckPvcHasGivenStatusCondition(svcClient, svcNamespace, pvcName, conditionsPresent, condition) +} + +// checkPvcHasGivenStatusCondition checks if the status condition in PVC +// matches with the one we want. +func CheckPvcHasGivenStatusCondition(client clientset.Interface, namespace string, pvcName string, + conditionsPresent bool, condition v1.PersistentVolumeClaimConditionType) (*v1.PersistentVolumeClaim, error) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + pvclaim, err := client.CoreV1().PersistentVolumeClaims(namespace).Get(ctx, pvcName, metav1.GetOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(pvclaim).NotTo(gomega.BeNil()) + inProgressConditions := pvclaim.Status.Conditions + + if len(inProgressConditions) == 0 { + if conditionsPresent { + return pvclaim, fmt.Errorf("no status conditions found on PVC: %v", pvcName) + } + return pvclaim, nil + } + conditionsMatches := false + for i := range inProgressConditions { + if inProgressConditions[i].Type == condition { + conditionsMatches = true + break + } + } + if conditionsMatches == conditionsPresent { + framework.Logf("PVC '%v' is in '%v' status condition", pvcName, condition) + } else { + return pvclaim, fmt.Errorf( + "status conditions found on PVC '%v' are not matching with expected status condition '%v'", + pvcName, condition) + } + return pvclaim, nil +} + +// waitForPVCToReachFileSystemResizePendingCondition waits for PVC to reach +// FileSystemResizePendingCondition status condition. +func WaitForPVCToReachFileSystemResizePendingCondition(client clientset.Interface, + namespace string, pvcName string, timeout time.Duration) (*v1.PersistentVolumeClaim, error) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + var pvclaim *v1.PersistentVolumeClaim + var err error + + waitErr := wait.PollUntilContextTimeout(ctx, constants.ResizePollInterval, timeout, true, + func(ctx context.Context) (bool, error) { + pvclaim, err = client.CoreV1().PersistentVolumeClaims(namespace).Get(ctx, pvcName, metav1.GetOptions{}) + if err != nil { + return false, fmt.Errorf("error while fetching pvc '%v' after controller resize: %v", pvcName, err) + } + gomega.Expect(pvclaim).NotTo(gomega.BeNil()) + + inProgressConditions := pvclaim.Status.Conditions + // If there are conditions on the PVC, it must be of + // FileSystemResizePending type. + resizeConditionFound := false + for i := range inProgressConditions { + if inProgressConditions[i].Type == v1.PersistentVolumeClaimFileSystemResizePending { + resizeConditionFound = true + } + } + if resizeConditionFound { + framework.Logf("PVC '%v' is in 'FileSystemResizePending' status condition", pvcName) + return true, nil + } else { + return false, fmt.Errorf("resize was not triggered on PVC '%v' or no status conditions related to "+ + "FileSystemResizePending found on it", pvcName) + } + }) + return pvclaim, waitErr +} diff --git a/tests/e2e/restore_on_target_ds/junit.xml b/tests/e2e/restore_on_target_ds/junit.xml new file mode 100644 index 0000000000..f67f0a121f --- /dev/null +++ b/tests/e2e/restore_on_target_ds/junit.xml @@ -0,0 +1,1163 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + [PANICKED] Test Panicked In [It] at: /Users/nayakb/vsphere-csi-driver/vendor/k8s.io/apimachinery/pkg/util/runtime/runtime.go:114 @ 09/30/25 13:43:07.307 runtime error: invalid memory address or nil pointer dereference Full Stack Trace k8s.io/apimachinery/pkg/util/runtime.handleCrash({0x1047ca458, 0x14001cc1950}, {0x103fd9a20, 0x106854f10}, {0x0, 0x0, 0x0}) /Users/nayakb/vsphere-csi-driver/vendor/k8s.io/apimachinery/pkg/util/runtime/runtime.go:114 +0x1b4 k8s.io/apimachinery/pkg/util/runtime.HandleCrashWithContext({0x1047ca500, 0x14000646f50}, {0x0, 0x0, 0x0}) /Users/nayakb/vsphere-csi-driver/vendor/k8s.io/apimachinery/pkg/util/runtime/runtime.go:78 +0x70 panic({0x103fd9a20?, 0x106854f10?}) /Users/nayakb/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.24.2.darwin-arm64/src/runtime/panic.go:792 +0xf0 sigs.k8s.io/vsphere-csi-driver/v3/tests/e2e/vcutil.QueryCNSVolumeSnapshotWithResult(0x14000774660, {0x14001335770, 0x24}, {0x14000b05885, 0x24}) /Users/nayakb/vsphere-csi-driver/tests/e2e/vcutil/vc_util.go:357 +0x338 sigs.k8s.io/vsphere-csi-driver/v3/tests/e2e/vcutil.VerifySnapshotIsCreatedInCNS(0x14000774660, {0x14001335770, 0x24}, {0x14000b05885, 0x24}) /Users/nayakb/vsphere-csi-driver/tests/e2e/vcutil/vc_util.go:397 +0x158 sigs.k8s.io/vsphere-csi-driver/v3/tests/e2e/csisnapshot.WaitForCNSSnapshotToBeCreated.func1({0x1047ca500, 0x14000646f50}) /Users/nayakb/vsphere-csi-driver/tests/e2e/csisnapshot/util.go:75 +0x74 k8s.io/apimachinery/pkg/util/wait.loopConditionUntilContext.func1({0x1047ca500, 0x14000646f50}, 0x14001d8f510) /Users/nayakb/vsphere-csi-driver/vendor/k8s.io/apimachinery/pkg/util/wait/loop.go:53 +0xcc k8s.io/apimachinery/pkg/util/wait.loopConditionUntilContext({0x1047ca500, 0x14000646f50}, {0x1047b4f20, 0x140007d7440}, 0x1, 0x0, 0x14001d8f510) /Users/nayakb/vsphere-csi-driver/vendor/k8s.io/apimachinery/pkg/util/wait/loop.go:54 +0x130 k8s.io/apimachinery/pkg/util/wait.PollUntilContextTimeout({0x1047ca538, 0x106901dc8}, 0x77359400, 0x1176592e000, 0x1, 0x14001d8f510) /Users/nayakb/vsphere-csi-driver/vendor/k8s.io/apimachinery/pkg/util/wait/poll.go:48 +0x100 sigs.k8s.io/vsphere-csi-driver/v3/tests/e2e/csisnapshot.WaitForCNSSnapshotToBeCreated(0x14000774660, {0x14001335770, 0x24}, {0x14000b05885, 0x24}) /Users/nayakb/vsphere-csi-driver/tests/e2e/csisnapshot/util.go:73 +0x17c sigs.k8s.io/vsphere-csi-driver/v3/tests/e2e/csisnapshot.CreateDynamicVolumeSnapshot({0x1047ca490, 0x14000194aa0}, 0x14000774660, {0x14001268540, 0x8}, 0x14001300960, 0x14000885180, 0x14000ba83c0, {0x14001335770, 0x24}, ...) /Users/nayakb/vsphere-csi-driver/tests/e2e/csisnapshot/util.go:231 +0x8ac sigs.k8s.io/vsphere-csi-driver/v3/tests/e2e/csisnapshot.CreateVolumeSnapshot({0x1047ca490, 0x14000194aa0}, 0x14000774660, {0x14001268540, 0x8}, 0x14000ba83c0, {0x140000ba560, 0x1, 0x1}, {0x10363bc38, ...}) /Users/nayakb/vsphere-csi-driver/tests/e2e/csisnapshot/util.go:416 +0x22c sigs.k8s.io/vsphere-csi-driver/v3/tests/e2e/restore_on_target_ds.init.func2.3() /Users/nayakb/vsphere-csi-driver/tests/e2e/restore_on_target_ds/restore_on_target_ds_vms_tests.go:182 +0x614 There were additional failures detected after the initial failure. These are visible in the timeline + > Enter [BeforeEach] [restore-on-target-ds-vms-p0] restore-on-target-ds-vms-p0 - set up framework | framework.go:200 @ 09/30/25 13:37:24.106 STEP: Creating a kubernetes client - /Users/nayakb/vsphere-csi-driver/vendor/k8s.io/kubernetes/test/e2e/framework/framework.go:220 @ 09/30/25 13:37:24.106 I0930 13:37:24.106580 67684 util.go:454] >>> kubeConfig: /Users/nayakb/vsphere-csi-driver/supervisor-cluster-kube-config-file < Exit [BeforeEach] [restore-on-target-ds-vms-p0] restore-on-target-ds-vms-p0 - set up framework | framework.go:200 @ 09/30/25 13:37:24.11 (4ms) > Enter [BeforeEach] [restore-on-target-ds-vms-p0] restore-on-target-ds-vms-p0 - /Users/nayakb/vsphere-csi-driver/tests/e2e/restore_on_target_ds/restore_on_target_ds_vms_tests.go:71 @ 09/30/25 13:37:24.11 I0930 13:37:26.454073 67684 util.go:6215] Running command: curl -k -X POST https://10.192.73.138/rest/com/vmware/cis/session -u 'Administrator@vsphere.local:FVZOIhzt6_kR0x_k' I0930 13:37:29.476433 67684 ssh.go:363] ssh root@10.192.73.138:22: command: curl -k -X POST https://10.192.73.138/rest/com/vmware/cis/session -u 'Administrator@vsphere.local:FVZOIhzt6_kR0x_k' I0930 13:37:29.476812 67684 ssh.go:364] ssh root@10.192.73.138:22: stdout: "{\"value\":\"f1240905bb4e401c7ab2a8ebb3139ab2\"}" I0930 13:37:29.476888 67684 ssh.go:365] ssh root@10.192.73.138:22: stderr: " % Total % Received % Xferd Average Speed Time Time Time Current\n Dload Upload Total Spent Left Speed\n\r 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0\r100 44 0 44 0 0 759 0 --:--:-- --:--:-- --:--:-- 771\n" I0930 13:37:29.476915 67684 ssh.go:366] ssh root@10.192.73.138:22: exit code: 0 I0930 13:37:29.477003 67684 util.go:6228] sessionID is: f1240905bb4e401c7ab2a8ebb3139ab2 I0930 13:37:31.196116 67684 vc_util.go:597] storage policy id: fcccd46d-c8a9-48a9-8c0d-5c34536fdfc5 for storage policy name is: vsan-thin I0930 13:37:33.107684 67684 restore_on_target_ds_vms_tests.go:102] dsmoId: datastore-33 I0930 13:37:33.107860 67684 vmservice_utils.go:290] Invoking GET on url: https://10.192.73.138/api/content/subscribed-library I0930 13:37:34.462290 67684 util.go:1541] API Response status 200 I0930 13:37:34.462414 67684 vmservice_utils.go:258] Content lib IDs: [e8f48107-e967-440f-846f-2c70d01869e7 72112bf5-5355-4987-ae7d-e2cf1a8f95fb] I0930 13:37:34.462468 67684 vmservice_utils.go:290] Invoking GET on url: https://10.192.73.138/api/content/subscribed-library/e8f48107-e967-440f-846f-2c70d01869e7 I0930 13:37:35.381030 67684 util.go:1541] API Response status 200 I0930 13:37:35.381306 67684 vmservice_utils.go:279] Content lib with id e8f48107-e967-440f-846f-2c70d01869e7: {e8f48107-e967-440f-846f-2c70d01869e7 Kubernetes Service Content Library datastore-111 https://wp-content.vmware.com/v2/latest/lib.json} I0930 13:37:35.381334 67684 vmservice_utils.go:290] Invoking GET on url: https://10.192.73.138/api/content/subscribed-library/72112bf5-5355-4987-ae7d-e2cf1a8f95fb I0930 13:37:36.302344 67684 util.go:1541] API Response status 200 I0930 13:37:36.302508 67684 vmservice_utils.go:279] Content lib with id 72112bf5-5355-4987-ae7d-e2cf1a8f95fb: {72112bf5-5355-4987-ae7d-e2cf1a8f95fb csi-vmsvc-8678 datastore-33 https://wp-content-pstg.broadcom.com/vmsvc/lib.json} I0930 13:37:36.304208 67684 util.go:7843] Invoking GET on url: https://10.192.73.138:443/api/vcenter/namespace-management/supervisors/summaries I0930 13:37:37.375957 67684 util.go:1541] API Response status 200 I0930 13:37:37.377706 67684 util.go:7832] Supervisor summary: map[items:[map[info:map[config_status:RUNNING kubernetes_status:READY name:supervisor stats:map[cpu_capacity:620760 cpu_used:10374 memory_capacity:631986 memory_used:57896 storage_capacity:151422 storage_used:54188]] supervisor:2f6bd7e7-b709-4c71-8356-4115abdb9d69]]] I0930 13:37:37.377912 67684 util.go:7787] Invoking POST on url: https://10.192.73.138:443/api/vcenter/namespaces/instances/v2 I0930 13:37:38.649651 67684 util.go:1541] API Response status 204 < Exit [BeforeEach] [restore-on-target-ds-vms-p0] restore-on-target-ds-vms-p0 - /Users/nayakb/vsphere-csi-driver/tests/e2e/restore_on_target_ds/restore_on_target_ds_vms_tests.go:71 @ 09/30/25 13:38:38.65 (1m14.54s) > Enter [It] Attach VM to restored PVC which already attached to original PVC - /Users/nayakb/vsphere-csi-driver/tests/e2e/restore_on_target_ds/restore_on_target_ds_vms_tests.go:158 @ 09/30/25 13:38:38.65 I0930 13:38:38.650378 67684 restore_on_target_ds_vms_tests.go:163] Starting test: Attach VM to restored PVC which already attached to original PVC STEP: Create PVC - /Users/nayakb/vsphere-csi-driver/tests/e2e/k8testutil/util.go:7927 @ 09/30/25 13:38:38.65 STEP: Creating PVC using the Storage Class vsan-thin with disk size and labels: map[] accessMode: - /Users/nayakb/vsphere-csi-driver/tests/e2e/k8testutil/util.go:984 @ 09/30/25 13:38:38.65 I0930 13:38:39.125430 67684 util.go:988] PVC created: pvc-7gw6b in namespace: csi-1613 I0930 13:38:39.125469 67684 pv.go:790] Waiting up to timeout=5m0s for PersistentVolumeClaims [pvc-7gw6b] to have phase Bound I0930 13:38:39.354350 67684 pv.go:806] PersistentVolumeClaim pvc-7gw6b found but phase is Pending instead of Bound. I0930 13:38:41.587954 67684 pv.go:801] PersistentVolumeClaim pvc-7gw6b found and phase=Bound (2.46246s) STEP: Create vm service vm - /Users/nayakb/vsphere-csi-driver/tests/e2e/restore_on_target_ds/restore_on_target_ds_vms_tests.go:168 @ 09/30/25 13:38:42.054 I0930 13:38:42.055040 67684 vmservice_utils.go:1291] Waiting for virtual machine image list to be available in namespace 'csi-1613' for image 'ubuntu-2004-cloud-init-21.4-kube-v1.20.10' I0930 13:39:14.515009 67684 vmservice_utils.go:343] Found vmi vmi-906e064069661b971 for image name ubuntu-2004-cloud-init-21.4-kube-v1.20.10 I0930 13:39:15.528287 67684 vmservice_utils.go:511] Found VM csi-test-vm-4291 in namespace csi-1613 I0930 13:39:15.528442 67684 vmservice_utils.go:561] Creating loadbalancer VM: csi-test-vm-4291-svc-4189 for vm: csi-test-vm-4291 I0930 13:42:56.269548 67684 vmservice_utils.go:533] Found IP '166.168.16.9' for VM 'csi-test-vm-4291' I0930 13:42:56.971523 67684 vmservice_utils.go:654] Given PVCs '[pvc-7gw6b]' are attached to VM csi-test-vm-4291 STEP: Wait for VM to come up and get an IP - /Users/nayakb/vsphere-csi-driver/tests/e2e/restore_on_target_ds/restore_on_target_ds_vms_tests.go:173 @ 09/30/25 13:42:57.215 I0930 13:42:57.456423 67684 vmservice_utils.go:533] Found IP '166.168.16.9' for VM 'csi-test-vm-4291' STEP: Wait and verify PVCs are attached to the VM - /Users/nayakb/vsphere-csi-driver/tests/e2e/restore_on_target_ds/restore_on_target_ds_vms_tests.go:177 @ 09/30/25 13:42:57.456 I0930 13:42:57.923598 67684 vmservice_utils.go:654] Given PVCs '[pvc-7gw6b]' are attached to VM csi-test-vm-4291 STEP: Get or create volume snapshot class - /Users/nayakb/vsphere-csi-driver/tests/e2e/csisnapshot/util.go:404 @ 09/30/25 13:42:57.924 I0930 13:42:58.159831 67684 util.go:320] volumesnapshotclass &{{ } {volumesnapshotclass-delete 856e3a6e-e0b3-4be2-aa00-769a206b22e6 1793 1 2025-09-23 15:45:19 +0530 IST <nil> <nil> map[] map[kubectl.kubernetes.io/last-applied-configuration:{"apiVersion":"snapshot.storage.k8s.io/v1","deletionPolicy":"Delete","driver":"csi.vsphere.vmware.com","kind":"VolumeSnapshotClass","metadata":{"annotations":{"snapshot.storage.kubernetes.io/is-default-class":"true"},"name":"volumesnapshotclass-delete"}} snapshot.storage.kubernetes.io/is-default-class:true] [] [] [{kubectl-client-side-apply Update snapshot.storage.k8s.io/v1 2025-09-23 15:45:19 +0530 IST FieldsV1 {"f:deletionPolicy":{},"f:driver":{},"f:metadata":{"f:annotations":{".":{},"f:kubectl.kubernetes.io/last-applied-configuration":{},"f:snapshot.storage.kubernetes.io/is-default-class":{}}}} }]} csi.vsphere.vmware.com map[] Delete}, err:<nil> I0930 13:42:58.160000 67684 util.go:326] Found volumesnapshotclass volumesnapshotclass-delete STEP: Create a volume snapshot - /Users/nayakb/vsphere-csi-driver/tests/e2e/csisnapshot/util.go:410 @ 09/30/25 13:42:58.16 I0930 13:42:58.589482 67684 util.go:197] Volume snapshot name is : snapshot-5v64w STEP: Verify volume snapshot is created - /Users/nayakb/vsphere-csi-driver/tests/e2e/csisnapshot/util.go:199 @ 09/30/25 13:42:58.589 STEP: Verify volume snapshot content is created - /Users/nayakb/vsphere-csi-driver/tests/e2e/csisnapshot/util.go:210 @ 09/30/25 13:43:06.834 I0930 13:43:07.305493 67684 util.go:57] volumesnapshotcontent details: &{{ } {snapcontent-4fed1aaf-ff6f-4c9f-aa19-bbde4e724234 e5cdc518-3976-408a-a95e-fddcdb912a79 7480477 1 2025-09-30 13:42:58 +0530 IST <nil> <nil> map[] map[] [] [snapshot.storage.kubernetes.io/volumesnapshotcontent-bound-protection] [{snapshot-controller Update snapshot.storage.k8s.io/v1 2025-09-30 13:42:58 +0530 IST FieldsV1 {"f:metadata":{"f:finalizers":{".":{},"v:\"snapshot.storage.kubernetes.io/volumesnapshotcontent-bound-protection\"":{}}},"f:spec":{".":{},"f:deletionPolicy":{},"f:driver":{},"f:source":{".":{},"f:volumeHandle":{}},"f:sourceVolumeMode":{},"f:volumeSnapshotClassName":{},"f:volumeSnapshotRef":{}}} } {csi-snapshotter Update snapshot.storage.k8s.io/v1 2025-09-30 13:43:05 +0530 IST FieldsV1 {"f:status":{".":{},"f:creationTime":{},"f:readyToUse":{},"f:restoreSize":{},"f:snapshotHandle":{}}} status}]} {{VolumeSnapshot csi-1613 snapshot-5v64w 4fed1aaf-ff6f-4c9f-aa19-bbde4e724234 snapshot.storage.k8s.io/v1 7480290 } Delete csi.vsphere.vmware.com 0x14000273d40 {0x14000273d20 <nil>} 0x14000273d30} 0x14001cc16b0} I0930 13:43:07.305717 67684 util.go:62] snapcontent-4fed1aaf-ff6f-4c9f-aa19-bbde4e724234 volume snapshotContent is in ready state I0930 13:43:07.305757 67684 util.go:223] Get volume snapshot ID from snapshot handle STEP: Query CNS and check the volume snapshot entry - /Users/nayakb/vsphere-csi-driver/tests/e2e/csisnapshot/util.go:230 @ 09/30/25 13:43:07.305 STEP: Invoking queryCNSVolumeSnapshotWithResult with VolumeID: 3507ee2e-c882-453c-9688-9d700e6bb97b and SnapshotID: d2ab02c3-eb08-4876-bf40-29b29605b2af - /Users/nayakb/vsphere-csi-driver/tests/e2e/vcutil/vc_util.go:393 @ 09/30/25 13:43:07.306 E0930 13:43:07.306798 67684 panic.go:262] "Observed a panic" panic="runtime error: invalid memory address or nil pointer dereference" panicGoValue="\"invalid memory address or nil pointer dereference\"" stacktrace=< goroutine 175 [running]: k8s.io/apimachinery/pkg/util/runtime.logPanic({0x1047ca458, 0x14001cc1950}, {0x103fd9a20, 0x106854f10}) /Users/nayakb/vsphere-csi-driver/vendor/k8s.io/apimachinery/pkg/util/runtime/runtime.go:132 +0xe4 k8s.io/apimachinery/pkg/util/runtime.handleCrash({0x1047ca458, 0x14001cc1950}, {0x103fd9a20, 0x106854f10}, {0x0, 0x0, 0x0}) /Users/nayakb/vsphere-csi-driver/vendor/k8s.io/apimachinery/pkg/util/runtime/runtime.go:107 +0x104 k8s.io/apimachinery/pkg/util/runtime.HandleCrashWithContext({0x1047ca500, 0x14000646f50}, {0x0, 0x0, 0x0}) /Users/nayakb/vsphere-csi-driver/vendor/k8s.io/apimachinery/pkg/util/runtime/runtime.go:78 +0x70 panic({0x103fd9a20?, 0x106854f10?}) /Users/nayakb/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.24.2.darwin-arm64/src/runtime/panic.go:792 +0xf0 sigs.k8s.io/vsphere-csi-driver/v3/tests/e2e/vcutil.QueryCNSVolumeSnapshotWithResult(0x14000774660, {0x14001335770, 0x24}, {0x14000b05885, 0x24}) /Users/nayakb/vsphere-csi-driver/tests/e2e/vcutil/vc_util.go:357 +0x338 sigs.k8s.io/vsphere-csi-driver/v3/tests/e2e/vcutil.VerifySnapshotIsCreatedInCNS(0x14000774660, {0x14001335770, 0x24}, {0x14000b05885, 0x24}) /Users/nayakb/vsphere-csi-driver/tests/e2e/vcutil/vc_util.go:397 +0x158 sigs.k8s.io/vsphere-csi-driver/v3/tests/e2e/csisnapshot.WaitForCNSSnapshotToBeCreated.func1({0x1047ca500, 0x14000646f50}) /Users/nayakb/vsphere-csi-driver/tests/e2e/csisnapshot/util.go:75 +0x74 k8s.io/apimachinery/pkg/util/wait.loopConditionUntilContext.func1({0x1047ca500, 0x14000646f50}, 0x14001d8f510) /Users/nayakb/vsphere-csi-driver/vendor/k8s.io/apimachinery/pkg/util/wait/loop.go:53 +0xcc k8s.io/apimachinery/pkg/util/wait.loopConditionUntilContext({0x1047ca500, 0x14000646f50}, {0x1047b4f20, 0x140007d7440}, 0x1, 0x0, 0x14001d8f510) /Users/nayakb/vsphere-csi-driver/vendor/k8s.io/apimachinery/pkg/util/wait/loop.go:54 +0x130 k8s.io/apimachinery/pkg/util/wait.PollUntilContextTimeout({0x1047ca538, 0x106901dc8}, 0x77359400, 0x1176592e000, 0x1, 0x14001d8f510) /Users/nayakb/vsphere-csi-driver/vendor/k8s.io/apimachinery/pkg/util/wait/poll.go:48 +0x100 sigs.k8s.io/vsphere-csi-driver/v3/tests/e2e/csisnapshot.WaitForCNSSnapshotToBeCreated(0x14000774660, {0x14001335770, 0x24}, {0x14000b05885, 0x24}) /Users/nayakb/vsphere-csi-driver/tests/e2e/csisnapshot/util.go:73 +0x17c sigs.k8s.io/vsphere-csi-driver/v3/tests/e2e/csisnapshot.CreateDynamicVolumeSnapshot({0x1047ca490, 0x14000194aa0}, 0x14000774660, {0x14001268540, 0x8}, 0x14001300960, 0x14000885180, 0x14000ba83c0, {0x14001335770, 0x24}, ...) /Users/nayakb/vsphere-csi-driver/tests/e2e/csisnapshot/util.go:231 +0x8ac sigs.k8s.io/vsphere-csi-driver/v3/tests/e2e/csisnapshot.CreateVolumeSnapshot({0x1047ca490, 0x14000194aa0}, 0x14000774660, {0x14001268540, 0x8}, 0x14000ba83c0, {0x140000ba560, 0x1, 0x1}, {0x10363bc38, ...}) /Users/nayakb/vsphere-csi-driver/tests/e2e/csisnapshot/util.go:416 +0x22c sigs.k8s.io/vsphere-csi-driver/v3/tests/e2e/restore_on_target_ds.init.func2.3() /Users/nayakb/vsphere-csi-driver/tests/e2e/restore_on_target_ds/restore_on_target_ds_vms_tests.go:182 +0x614 github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func3({0x1047cf988, 0x14000b36cc0}) /Users/nayakb/vsphere-csi-driver/vendor/github.com/onsi/ginkgo/v2/internal/node.go:475 +0x30 github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3() /Users/nayakb/vsphere-csi-driver/vendor/github.com/onsi/ginkgo/v2/internal/suite.go:894 +0x17c created by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 118 /Users/nayakb/vsphere-csi-driver/vendor/github.com/onsi/ginkgo/v2/internal/suite.go:881 +0xd94 > [PANICKED] Test Panicked In [It] at: /Users/nayakb/vsphere-csi-driver/vendor/k8s.io/apimachinery/pkg/util/runtime/runtime.go:114 @ 09/30/25 13:43:07.307 runtime error: invalid memory address or nil pointer dereference Full Stack Trace k8s.io/apimachinery/pkg/util/runtime.handleCrash({0x1047ca458, 0x14001cc1950}, {0x103fd9a20, 0x106854f10}, {0x0, 0x0, 0x0}) /Users/nayakb/vsphere-csi-driver/vendor/k8s.io/apimachinery/pkg/util/runtime/runtime.go:114 +0x1b4 k8s.io/apimachinery/pkg/util/runtime.HandleCrashWithContext({0x1047ca500, 0x14000646f50}, {0x0, 0x0, 0x0}) /Users/nayakb/vsphere-csi-driver/vendor/k8s.io/apimachinery/pkg/util/runtime/runtime.go:78 +0x70 panic({0x103fd9a20?, 0x106854f10?}) /Users/nayakb/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.24.2.darwin-arm64/src/runtime/panic.go:792 +0xf0 sigs.k8s.io/vsphere-csi-driver/v3/tests/e2e/vcutil.QueryCNSVolumeSnapshotWithResult(0x14000774660, {0x14001335770, 0x24}, {0x14000b05885, 0x24}) /Users/nayakb/vsphere-csi-driver/tests/e2e/vcutil/vc_util.go:357 +0x338 sigs.k8s.io/vsphere-csi-driver/v3/tests/e2e/vcutil.VerifySnapshotIsCreatedInCNS(0x14000774660, {0x14001335770, 0x24}, {0x14000b05885, 0x24}) /Users/nayakb/vsphere-csi-driver/tests/e2e/vcutil/vc_util.go:397 +0x158 sigs.k8s.io/vsphere-csi-driver/v3/tests/e2e/csisnapshot.WaitForCNSSnapshotToBeCreated.func1({0x1047ca500, 0x14000646f50}) /Users/nayakb/vsphere-csi-driver/tests/e2e/csisnapshot/util.go:75 +0x74 k8s.io/apimachinery/pkg/util/wait.loopConditionUntilContext.func1({0x1047ca500, 0x14000646f50}, 0x14001d8f510) /Users/nayakb/vsphere-csi-driver/vendor/k8s.io/apimachinery/pkg/util/wait/loop.go:53 +0xcc k8s.io/apimachinery/pkg/util/wait.loopConditionUntilContext({0x1047ca500, 0x14000646f50}, {0x1047b4f20, 0x140007d7440}, 0x1, 0x0, 0x14001d8f510) /Users/nayakb/vsphere-csi-driver/vendor/k8s.io/apimachinery/pkg/util/wait/loop.go:54 +0x130 k8s.io/apimachinery/pkg/util/wait.PollUntilContextTimeout({0x1047ca538, 0x106901dc8}, 0x77359400, 0x1176592e000, 0x1, 0x14001d8f510) /Users/nayakb/vsphere-csi-driver/vendor/k8s.io/apimachinery/pkg/util/wait/poll.go:48 +0x100 sigs.k8s.io/vsphere-csi-driver/v3/tests/e2e/csisnapshot.WaitForCNSSnapshotToBeCreated(0x14000774660, {0x14001335770, 0x24}, {0x14000b05885, 0x24}) /Users/nayakb/vsphere-csi-driver/tests/e2e/csisnapshot/util.go:73 +0x17c sigs.k8s.io/vsphere-csi-driver/v3/tests/e2e/csisnapshot.CreateDynamicVolumeSnapshot({0x1047ca490, 0x14000194aa0}, 0x14000774660, {0x14001268540, 0x8}, 0x14001300960, 0x14000885180, 0x14000ba83c0, {0x14001335770, 0x24}, ...) /Users/nayakb/vsphere-csi-driver/tests/e2e/csisnapshot/util.go:231 +0x8ac sigs.k8s.io/vsphere-csi-driver/v3/tests/e2e/csisnapshot.CreateVolumeSnapshot({0x1047ca490, 0x14000194aa0}, 0x14000774660, {0x14001268540, 0x8}, 0x14000ba83c0, {0x140000ba560, 0x1, 0x1}, {0x10363bc38, ...}) /Users/nayakb/vsphere-csi-driver/tests/e2e/csisnapshot/util.go:416 +0x22c sigs.k8s.io/vsphere-csi-driver/v3/tests/e2e/restore_on_target_ds.init.func2.3() /Users/nayakb/vsphere-csi-driver/tests/e2e/restore_on_target_ds/restore_on_target_ds_vms_tests.go:182 +0x614 < Exit [It] Attach VM to restored PVC which already attached to original PVC - /Users/nayakb/vsphere-csi-driver/tests/e2e/restore_on_target_ds/restore_on_target_ds_vms_tests.go:158 @ 09/30/25 13:43:07.307 (4m28.659s) > Enter [AfterEach] [restore-on-target-ds-vms-p0] restore-on-target-ds-vms-p0 - /Users/nayakb/vsphere-csi-driver/tests/e2e/restore_on_target_ds/restore_on_target_ds_vms_tests.go:139 @ 09/30/25 13:43:07.307 I0930 13:43:07.307616 67684 restore_on_target_ds_vms_tests.go:142] Collecting supervisor PVC events before performing PV/PVC cleanup STEP: Found 43 events in svc ns csi-1613. - /Users/nayakb/vsphere-csi-driver/tests/e2e/k8testutil/util.go:6674 @ 09/30/25 13:43:07.769 I0930 13:43:07.769493 67684 util.go:6680] At 2025-09-30 13:37:41 +0530 IST - event for cl-6ddd5341d34788d3b: {vmware-system-imageregistry/vmware-system-imageregistry-controller-manager/contentlibrary-controller } UpdateStatusSuccess: UpdateStatus success I0930 13:43:07.769547 67684 util.go:6680] At 2025-09-30 13:37:43 +0530 IST - event for cl-6ddd5341d34788d3b: {vmware-system-imageregistry/vmware-system-imageregistry-controller-manager/contentlibrary-controller } CreateItemsSuccess: CreateItems success I0930 13:43:07.769568 67684 util.go:6680] At 2025-09-30 13:37:44 +0530 IST - event for clitem-c5f07dddc6c325439: {vmware-system-imageregistry/vmware-system-imageregistry-controller-manager } UpdateStatusSuccess: UpdateStatus success I0930 13:43:07.769597 67684 util.go:6680] At 2025-09-30 13:37:45 +0530 IST - event for clitem-1a05a30a92046e657: {vmware-system-imageregistry/vmware-system-imageregistry-controller-manager } UpdateStatusSuccess: UpdateStatus success I0930 13:43:07.769619 67684 util.go:6680] At 2025-09-30 13:37:45 +0530 IST - event for clitem-8f9cd0b4fb85ced8e: {vmware-system-imageregistry/vmware-system-imageregistry-controller-manager } UpdateStatusSuccess: UpdateStatus success I0930 13:43:07.769639 67684 util.go:6680] At 2025-09-30 13:37:45 +0530 IST - event for clitem-e9c170938cf34b721: {vmware-system-imageregistry/vmware-system-imageregistry-controller-manager } UpdateStatusSuccess: UpdateStatus success I0930 13:43:07.769661 67684 util.go:6680] At 2025-09-30 13:37:45 +0530 IST - event for vmi-021803e1e74d98e91: {vmware-system-vmop/vmware-system-vmop-controller-manager-89fd58b47-26kkl/contentlibraryitem-controller } UpdateSuccess: Update success I0930 13:43:07.769684 67684 util.go:6680] At 2025-09-30 13:37:46 +0530 IST - event for clitem-70b232afd7d90a37e: {vmware-system-imageregistry/vmware-system-imageregistry-controller-manager } UpdateStatusSuccess: UpdateStatus success I0930 13:43:07.770057 67684 util.go:6680] At 2025-09-30 13:37:46 +0530 IST - event for clitem-b47bcc87a02ffacd4: {vmware-system-imageregistry/vmware-system-imageregistry-controller-manager } UpdateStatusSuccess: UpdateStatus success I0930 13:43:07.770090 67684 util.go:6680] At 2025-09-30 13:37:46 +0530 IST - event for clitem-b7d135c30e28717f5: {vmware-system-imageregistry/vmware-system-imageregistry-controller-manager } UpdateStatusSuccess: UpdateStatus success I0930 13:43:07.770109 67684 util.go:6680] At 2025-09-30 13:37:46 +0530 IST - event for clitem-bbdc5a8477c349751: {vmware-system-imageregistry/vmware-system-imageregistry-controller-manager } UpdateStatusSuccess: UpdateStatus success I0930 13:43:07.770123 67684 util.go:6680] At 2025-09-30 13:37:46 +0530 IST - event for clitem-bffefb69f8cef1df3: {vmware-system-imageregistry/vmware-system-imageregistry-controller-manager } UpdateStatusSuccess: UpdateStatus success I0930 13:43:07.770131 67684 util.go:6680] At 2025-09-30 13:37:46 +0530 IST - event for clitem-db6889ad36febd9f1: {vmware-system-imageregistry/vmware-system-imageregistry-controller-manager } UpdateStatusSuccess: UpdateStatus success I0930 13:43:07.770139 67684 util.go:6680] At 2025-09-30 13:37:47 +0530 IST - event for clitem-021803e1e74d98e91: {vmware-system-imageregistry/vmware-system-imageregistry-controller-manager } UpdateStatusSuccess: UpdateStatus success I0930 13:43:07.770146 67684 util.go:6680] At 2025-09-30 13:37:47 +0530 IST - event for clitem-282b43d4f1f6b89c1: {vmware-system-imageregistry/vmware-system-imageregistry-controller-manager } UpdateStatusSuccess: UpdateStatus success I0930 13:43:07.770152 67684 util.go:6680] At 2025-09-30 13:37:47 +0530 IST - event for clitem-906e064069661b971: {vmware-system-imageregistry/vmware-system-imageregistry-controller-manager } UpdateStatusSuccess: UpdateStatus success I0930 13:43:07.770158 67684 util.go:6680] At 2025-09-30 13:37:47 +0530 IST - event for clitem-b169727fa03df467b: {vmware-system-imageregistry/vmware-system-imageregistry-controller-manager } UpdateStatusSuccess: UpdateStatus success I0930 13:43:07.770165 67684 util.go:6680] At 2025-09-30 13:37:49 +0530 IST - event for vmi-c5f07dddc6c325439: {vmware-system-vmop/vmware-system-vmop-controller-manager-89fd58b47-26kkl/contentlibraryitem-controller } UpdateSuccess: Update success I0930 13:43:07.770171 67684 util.go:6680] At 2025-09-30 13:37:56 +0530 IST - event for vmi-e9c170938cf34b721: {vmware-system-vmop/vmware-system-vmop-controller-manager-89fd58b47-26kkl/contentlibraryitem-controller } UpdateSuccess: Update success I0930 13:43:07.770178 67684 util.go:6680] At 2025-09-30 13:38:08 +0530 IST - event for vmi-8f9cd0b4fb85ced8e: {vmware-system-vmop/vmware-system-vmop-controller-manager-89fd58b47-26kkl/contentlibraryitem-controller } UpdateSuccess: Update success I0930 13:43:07.770184 67684 util.go:6680] At 2025-09-30 13:38:15 +0530 IST - event for vmi-1a05a30a92046e657: {vmware-system-vmop/vmware-system-vmop-controller-manager-89fd58b47-26kkl/contentlibraryitem-controller } UpdateSuccess: Update success I0930 13:43:07.770191 67684 util.go:6680] At 2025-09-30 13:38:22 +0530 IST - event for vmi-b7d135c30e28717f5: {vmware-system-vmop/vmware-system-vmop-controller-manager-89fd58b47-26kkl/contentlibraryitem-controller } UpdateSuccess: Update success I0930 13:43:07.770197 67684 util.go:6680] At 2025-09-30 13:38:29 +0530 IST - event for vmi-b47bcc87a02ffacd4: {vmware-system-vmop/vmware-system-vmop-controller-manager-89fd58b47-26kkl/contentlibraryitem-controller } UpdateSuccess: Update success I0930 13:43:07.770204 67684 util.go:6680] At 2025-09-30 13:38:35 +0530 IST - event for vmi-db6889ad36febd9f1: {vmware-system-vmop/vmware-system-vmop-controller-manager-89fd58b47-26kkl/contentlibraryitem-controller } UpdateSuccess: Update success I0930 13:43:07.770212 67684 util.go:6680] At 2025-09-30 13:38:38 +0530 IST - event for pvc-7gw6b: {csi.vsphere.vmware.com_423c8fa2e527d5a5b917dc418171c3aa_f9302315-709e-47dd-a85d-3725fbcc40b5 } Provisioning: External provisioner is provisioning volume for claim "csi-1613/pvc-7gw6b" I0930 13:43:07.770222 67684 util.go:6680] At 2025-09-30 13:38:38 +0530 IST - event for pvc-7gw6b: {persistentvolume-controller } ExternalProvisioning: Waiting for a volume to be created either by the external provisioner 'csi.vsphere.vmware.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. I0930 13:43:07.770235 67684 util.go:6680] At 2025-09-30 13:38:40 +0530 IST - event for pvc-7gw6b: {csi.vsphere.vmware.com_423c8fa2e527d5a5b917dc418171c3aa_f9302315-709e-47dd-a85d-3725fbcc40b5 } ProvisioningSucceeded: Successfully provisioned volume pvc-bf114a3e-8274-40b8-99ba-7017e03ab22f I0930 13:43:07.770243 67684 util.go:6680] At 2025-09-30 13:38:42 +0530 IST - event for vmi-bbdc5a8477c349751: {vmware-system-vmop/vmware-system-vmop-controller-manager-89fd58b47-26kkl/contentlibraryitem-controller } UpdateSuccess: Update success I0930 13:43:07.770251 67684 util.go:6680] At 2025-09-30 13:38:49 +0530 IST - event for vmi-bffefb69f8cef1df3: {vmware-system-vmop/vmware-system-vmop-controller-manager-89fd58b47-26kkl/contentlibraryitem-controller } UpdateSuccess: Update success I0930 13:43:07.770257 67684 util.go:6680] At 2025-09-30 13:38:56 +0530 IST - event for vmi-70b232afd7d90a37e: {vmware-system-vmop/vmware-system-vmop-controller-manager-89fd58b47-26kkl/contentlibraryitem-controller } UpdateSuccess: Update success I0930 13:43:07.770263 67684 util.go:6680] At 2025-09-30 13:39:03 +0530 IST - event for vmi-b169727fa03df467b: {vmware-system-vmop/vmware-system-vmop-controller-manager-89fd58b47-26kkl/contentlibraryitem-controller } UpdateSuccess: Update success I0930 13:43:07.770270 67684 util.go:6680] At 2025-09-30 13:39:09 +0530 IST - event for vmi-906e064069661b971: {vmware-system-vmop/vmware-system-vmop-controller-manager-89fd58b47-26kkl/contentlibraryitem-controller } UpdateSuccess: Update success I0930 13:43:07.770276 67684 util.go:6680] At 2025-09-30 13:39:15 +0530 IST - event for csi-test-vm-4291-svc-4189: {vmware-system-vmop/vmware-system-vmop-controller-manager-89fd58b47-26kkl/virtualmachineservice-controller } CreateK8sServiceSuccess: CreateK8sService success I0930 13:43:07.770283 67684 util.go:6680] At 2025-09-30 13:39:15 +0530 IST - event for csi-test-vm-4291-svc-4189: {HAProxyGateway } Reconcile: Success I0930 13:43:07.770290 67684 util.go:6680] At 2025-09-30 13:39:16 +0530 IST - event for vmi-282b43d4f1f6b89c1: {vmware-system-vmop/vmware-system-vmop-controller-manager-89fd58b47-26kkl/contentlibraryitem-controller } UpdateSuccess: Update success I0930 13:43:07.770297 67684 util.go:6680] At 2025-09-30 13:42:04 +0530 IST - event for csi-test-vm-4291: {vmware-system-vmop/vmware-system-vmop-controller-manager-89fd58b47-26kkl/virtualmachine-controller } CreateSuccess: Create success I0930 13:43:07.770304 67684 util.go:6680] At 2025-09-30 13:42:04 +0530 IST - event for csi-test-vm-4291: {vmware-system-vmop/vmware-system-vmop-controller-manager-89fd58b47-26kkl/virtualmachine-controller } UpdateSuccess: Update success I0930 13:43:07.770311 67684 util.go:6680] At 2025-09-30 13:42:05 +0530 IST - event for csi-test-vm-4291: {vmware-system-vmop/vmware-system-vmop-controller-manager-89fd58b47-26kkl/virtualmachine-controller } UpdateFailure: failed to reconcile config: updating state failed with status update pending for persistent volume: pvc-7gw6b on VM I0930 13:43:07.770332 67684 util.go:6680] At 2025-09-30 13:42:06 +0530 IST - event for csi-test-vm-4291: {vmware-system-vmop/vmware-system-vmop-controller-manager-89fd58b47-26kkl/virtualmachine-controller } UpdateFailure: failed to reconcile config: updating state failed with persistent volume: pvc-7gw6b not attached to VM I0930 13:43:07.770340 67684 util.go:6680] At 2025-09-30 13:42:06 +0530 IST - event for csi-test-vm-4291-pvc-7gw6b: {cns.vmware.com } NodeVMAttachSucceeded: ReconcileCnsNodeVMAttachment: Successfully updated entry in CNS for instance with name "csi-test-vm-4291-pvc-7gw6b" and namespace "csi-1613". I0930 13:43:07.770347 67684 util.go:6680] At 2025-09-30 13:42:57 +0530 IST - event for snapshot-5v64w: {snapshot-controller } CreatingSnapshot: Waiting for a snapshot csi-1613/snapshot-5v64w to be created by the CSI driver. I0930 13:43:07.770549 67684 util.go:6680] At 2025-09-30 13:43:05 +0530 IST - event for snapshot-5v64w: {snapshot-controller } SnapshotReady: Snapshot csi-1613/snapshot-5v64w is ready to use. I0930 13:43:07.770563 67684 util.go:6680] At 2025-09-30 13:43:05 +0530 IST - event for snapshot-5v64w: {snapshot-controller } SnapshotCreated: Snapshot csi-1613/snapshot-5v64w was successfully created by the CSI driver. I0930 13:43:08.000527 67684 restore_on_target_ds_vms_tests.go:148] "UpdateStatus success" I0930 13:43:08.000586 67684 restore_on_target_ds_vms_tests.go:148] "CreateItems success" I0930 13:43:08.000595 67684 restore_on_target_ds_vms_tests.go:148] "UpdateStatus success" I0930 13:43:08.000603 67684 restore_on_target_ds_vms_tests.go:148] "UpdateStatus success" I0930 13:43:08.000610 67684 restore_on_target_ds_vms_tests.go:148] "UpdateStatus success" I0930 13:43:08.000616 67684 restore_on_target_ds_vms_tests.go:148] "UpdateStatus success" I0930 13:43:08.000623 67684 restore_on_target_ds_vms_tests.go:148] "UpdateStatus success" I0930 13:43:08.000630 67684 restore_on_target_ds_vms_tests.go:148] "UpdateStatus success" I0930 13:43:08.000637 67684 restore_on_target_ds_vms_tests.go:148] "UpdateStatus success" I0930 13:43:08.000644 67684 restore_on_target_ds_vms_tests.go:148] "UpdateStatus success" I0930 13:43:08.000650 67684 restore_on_target_ds_vms_tests.go:148] "UpdateStatus success" I0930 13:43:08.000656 67684 restore_on_target_ds_vms_tests.go:148] "UpdateStatus success" I0930 13:43:08.000664 67684 restore_on_target_ds_vms_tests.go:148] "UpdateStatus success" I0930 13:43:08.000671 67684 restore_on_target_ds_vms_tests.go:148] "UpdateStatus success" I0930 13:43:08.000677 67684 restore_on_target_ds_vms_tests.go:148] "UpdateStatus success" I0930 13:43:08.000683 67684 restore_on_target_ds_vms_tests.go:148] "UpdateStatus success" I0930 13:43:08.000694 67684 restore_on_target_ds_vms_tests.go:148] "ReconcileCnsNodeVMAttachment: Successfully updated entry in CNS for instance with name \"csi-test-vm-4291-pvc-7gw6b\" and namespace \"csi-1613\"." I0930 13:43:08.000701 67684 restore_on_target_ds_vms_tests.go:148] "CreateK8sService success" I0930 13:43:08.000707 67684 restore_on_target_ds_vms_tests.go:148] "Success" I0930 13:43:08.000716 67684 restore_on_target_ds_vms_tests.go:148] "Create success" I0930 13:43:08.000722 67684 restore_on_target_ds_vms_tests.go:148] "Update success" I0930 13:43:08.000730 67684 restore_on_target_ds_vms_tests.go:148] "failed to reconcile config: updating state failed with status update pending for persistent volume: pvc-7gw6b on VM" I0930 13:43:08.000737 67684 restore_on_target_ds_vms_tests.go:148] "failed to reconcile config: updating state failed with persistent volume: pvc-7gw6b not attached to VM" I0930 13:43:08.000747 67684 restore_on_target_ds_vms_tests.go:148] "Waiting for a volume to be created either by the external provisioner 'csi.vsphere.vmware.com' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered." I0930 13:43:08.000760 67684 restore_on_target_ds_vms_tests.go:148] "External provisioner is provisioning volume for claim \"csi-1613/pvc-7gw6b\"" I0930 13:43:08.000768 67684 restore_on_target_ds_vms_tests.go:148] "Successfully provisioned volume pvc-bf114a3e-8274-40b8-99ba-7017e03ab22f" I0930 13:43:08.000776 67684 restore_on_target_ds_vms_tests.go:148] "Waiting for a snapshot csi-1613/snapshot-5v64w to be created by the CSI driver." I0930 13:43:08.000784 67684 restore_on_target_ds_vms_tests.go:148] "Snapshot csi-1613/snapshot-5v64w was successfully created by the CSI driver." I0930 13:43:08.000791 67684 restore_on_target_ds_vms_tests.go:148] "Snapshot csi-1613/snapshot-5v64w is ready to use." I0930 13:43:08.000797 67684 restore_on_target_ds_vms_tests.go:148] "Update success" I0930 13:43:08.000804 67684 restore_on_target_ds_vms_tests.go:148] "Update success" I0930 13:43:08.000810 67684 restore_on_target_ds_vms_tests.go:148] "Update success" I0930 13:43:08.000817 67684 restore_on_target_ds_vms_tests.go:148] "Update success" I0930 13:43:08.000823 67684 restore_on_target_ds_vms_tests.go:148] "Update success" I0930 13:43:08.000829 67684 restore_on_target_ds_vms_tests.go:148] "Update success" I0930 13:43:08.000836 67684 restore_on_target_ds_vms_tests.go:148] "Update success" I0930 13:43:08.000842 67684 restore_on_target_ds_vms_tests.go:148] "Update success" I0930 13:43:08.000848 67684 restore_on_target_ds_vms_tests.go:148] "Update success" I0930 13:43:08.000855 67684 restore_on_target_ds_vms_tests.go:148] "Update success" I0930 13:43:08.000861 67684 restore_on_target_ds_vms_tests.go:148] "Update success" I0930 13:43:08.000867 67684 restore_on_target_ds_vms_tests.go:148] "Update success" I0930 13:43:08.000873 67684 restore_on_target_ds_vms_tests.go:148] "Update success" I0930 13:43:08.000879 67684 restore_on_target_ds_vms_tests.go:148] "Update success" I0930 13:43:08.000895 67684 util.go:7873] Invoking DELETE on url: https://10.192.73.138:443/api/vcenter/namespaces/instances/csi-1613 I0930 13:43:08.963473 67684 util.go:1541] API Response status 204 I0930 13:43:08.963644 67684 util.go:7864] Successfully Deleted namepsace csi-1613 in SVC. I0930 13:43:08.963674 67684 util.go:3886] Waiting up to 10m0s for namespace csi-1613 to get deleted I0930 13:43:09.198162 67684 util.go:3890] Namespace csi-1613 found and status={Terminating []} (234.431042ms) I0930 13:43:11.430321 67684 util.go:3890] Namespace csi-1613 found and status={Terminating []} (2.466607292s) I0930 13:43:13.678856 67684 util.go:3890] Namespace csi-1613 found and status={Terminating []} (4.715146625s) I0930 13:43:15.928349 67684 util.go:3890] Namespace csi-1613 found and status={Terminating []} (6.964696s) I0930 13:43:18.166387 67684 util.go:3890] Namespace csi-1613 found and status={Terminating []} (9.202724334s) I0930 13:43:21.065719 67684 util.go:3890] Namespace csi-1613 found and status={Terminating []} (12.102075292s) I0930 13:43:23.303145 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure True 2025-09-30 13:43:21 +0530 IST ContentDeletionFailed Failed to delete all resource types, 1 remaining: admission webhook "validation.csi.vsphere.vmware.com" denied the request: Deleting volume with snapshots is not allowed} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, volumesnapshots.snapshot.storage.k8s.io has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, snapshot.storage.kubernetes.io/volumesnapshot-bound-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (14.339435875s) I0930 13:43:25.535001 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure True 2025-09-30 13:43:21 +0530 IST ContentDeletionFailed Failed to delete all resource types, 1 remaining: admission webhook "validation.csi.vsphere.vmware.com" denied the request: Deleting volume with snapshots is not allowed} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, volumesnapshots.snapshot.storage.k8s.io has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, snapshot.storage.kubernetes.io/volumesnapshot-bound-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (16.571277375s) I0930 13:43:27.763468 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, kubernetes.io/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (18.799794667s) I0930 13:43:29.994002 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, kubernetes.io/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (21.030348375s) I0930 13:43:32.227454 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, kubernetes.io/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (23.263820584s) I0930 13:43:34.514159 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, kubernetes.io/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (25.550562584s) I0930 13:43:36.770360 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (27.806785167s) I0930 13:43:39.000321 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (30.036458334s) I0930 13:43:41.247509 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (32.283952334s) I0930 13:43:43.494546 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (34.531012042s) I0930 13:43:45.729444 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (36.765929417s) I0930 13:43:47.960842 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (38.997353292s) I0930 13:43:50.190513 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (41.227062125s) I0930 13:43:52.418872 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (43.455424542s) I0930 13:43:54.648190 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (45.684754667s) I0930 13:43:56.891224 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (47.927809292s) I0930 13:43:59.156499 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (50.193132959s) I0930 13:44:01.384943 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (52.421552167s) I0930 13:44:03.645015 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (54.68168975s) I0930 13:44:05.887455 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (56.92418s) I0930 13:44:08.121873 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (59.158540625s) I0930 13:44:10.354860 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (1m1.391592834s) I0930 13:44:12.603929 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (1m3.640678417s) I0930 13:44:15.079301 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (1m6.116110334s) I0930 13:44:17.308284 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (1m8.34511375s) I0930 13:44:19.568551 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (1m10.605396084s) I0930 13:44:21.840228 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (1m12.8769815s) I0930 13:44:24.089186 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (1m15.1260115s) I0930 13:44:26.387090 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (1m17.423794625s) I0930 13:44:28.629636 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (1m19.666510209s) I0930 13:44:30.874620 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (1m21.911224s) I0930 13:44:33.112143 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (1m24.1487845s) I0930 13:44:35.350859 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (1m26.387731542s) I0930 13:44:37.584933 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (1m28.621924167s) I0930 13:44:39.814936 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (1m30.851840709s) I0930 13:44:42.060831 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (1m33.097806084s) I0930 13:44:44.302942 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (1m35.339913042s) I0930 13:44:46.540920 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (1m37.5779935s) I0930 13:44:48.775544 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (1m39.81255s) I0930 13:44:51.013316 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (1m42.05035925s) I0930 13:44:53.256860 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (1m44.293730584s) I0930 13:44:55.492989 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (1m46.529960459s) I0930 13:44:57.725383 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (1m48.762509334s) I0930 13:44:59.964065 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (1m51.00118175s) I0930 13:45:02.221186 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (1m53.258304042s) I0930 13:45:04.463192 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (1m55.500346417s) I0930 13:45:06.704980 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (1m57.742136834s) I0930 13:45:08.941421 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (1m59.978586834s) I0930 13:45:11.170359 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (2m2.207596334s) I0930 13:45:13.414400 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (2m4.451696459s) I0930 13:45:15.651480 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (2m6.688725125s) I0930 13:45:17.892612 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (2m8.929950625s) I0930 13:45:20.135733 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (2m11.17303175s) I0930 13:45:22.366008 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (2m13.403275167s) I0930 13:45:24.594898 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (2m15.632201625s) I0930 13:45:26.824880 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (2m17.862243042s) I0930 13:45:29.071188 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (2m20.108611875s) I0930 13:45:31.329925 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (2m22.367316709s) I0930 13:45:33.567225 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (2m24.604614375s) I0930 13:45:35.815043 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (2m26.852475875s) I0930 13:45:38.055896 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (2m29.092716167s) I0930 13:45:40.312815 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (2m31.350332417s) I0930 13:45:42.546387 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (2m33.583795209s) I0930 13:45:44.778251 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (2m35.815581042s) I0930 13:45:47.019099 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (2m38.056686s) I0930 13:45:49.266544 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (2m40.304092459s) I0930 13:45:51.492994 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (2m42.53059575s) I0930 13:45:53.747070 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (2m44.78465275s) I0930 13:45:56.031140 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (2m47.068774209s) I0930 13:45:58.262494 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (2m49.299940167s) I0930 13:46:00.512647 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (2m51.550268792s) I0930 13:46:02.757938 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (2m53.795588417s) I0930 13:46:04.995501 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (2m56.033149167s) I0930 13:46:07.244574 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (2m58.282247459s) I0930 13:46:09.482283 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (3m0.519985042s) I0930 13:46:11.716826 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (3m2.75454775s) I0930 13:46:13.946670 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (3m4.984362459s) I0930 13:46:16.178427 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (3m7.216180792s) I0930 13:46:18.405436 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (3m9.443215375s) I0930 13:46:20.645200 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (3m11.682992459s) I0930 13:46:22.879458 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (3m13.917281042s) I0930 13:46:25.113104 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (3m16.150973459s) I0930 13:46:27.342970 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (3m18.380834917s) I0930 13:46:29.573715 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (3m20.611588292s) I0930 13:46:31.820607 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (3m22.857735667s) I0930 13:46:34.067489 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (3m25.105398084s) I0930 13:46:36.304429 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (3m27.342356917s) I0930 13:46:38.538489 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (3m29.576439s) I0930 13:46:40.774385 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (3m31.812367334s) I0930 13:46:43.008570 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (3m34.046546292s) I0930 13:46:45.242260 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (3m36.280220167s) I0930 13:46:47.498084 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (3m38.536101084s) I0930 13:46:49.737373 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (3m40.775474584s) I0930 13:46:51.983041 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (3m43.020476542s) I0930 13:46:54.213147 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (3m45.251305042s) I0930 13:46:56.460547 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (3m47.49865975s) I0930 13:46:58.770396 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (3m49.808568834s) I0930 13:47:01.026196 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (3m52.064340709s) I0930 13:47:03.278660 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (3m54.316887209s) I0930 13:47:05.524574 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (3m56.561997959s) I0930 13:47:07.802023 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (3m58.840254917s) I0930 13:47:10.040950 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (4m1.079265709s) I0930 13:47:12.275847 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (4m3.3106685s) I0930 13:47:14.511541 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (4m5.549790375s) I0930 13:47:16.752376 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (4m7.790484209s) I0930 13:47:18.980785 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (4m10.019181125s) I0930 13:47:21.218228 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (4m12.2564815s) I0930 13:47:23.459922 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (4m14.498269417s) I0930 13:47:25.697595 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (4m16.734096125s) I0930 13:47:27.933155 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (4m18.971543375s) I0930 13:47:30.170024 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (4m21.208476542s) I0930 13:47:32.399105 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (4m23.437561125s) I0930 13:47:34.630843 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (4m25.669299167s) I0930 13:47:36.861569 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (4m27.9000835s) I0930 13:47:39.097660 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (4m30.136193542s) I0930 13:47:41.335075 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (4m32.373625667s) I0930 13:47:43.564310 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (4m34.602817875s) I0930 13:47:45.804509 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (4m36.843008042s) I0930 13:47:48.033799 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (4m39.07236475s) I0930 13:47:50.270558 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (4m41.309110167s) I0930 13:47:52.514841 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (4m43.55327175s) I0930 13:47:54.756209 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (4m45.794835709s) I0930 13:47:56.994846 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (4m48.032641584s) I0930 13:47:59.230344 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (4m50.269039792s) I0930 13:48:01.468460 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (4m52.507020875s) I0930 13:48:03.700109 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (4m54.738788084s) I0930 13:48:05.930913 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (4m56.969324209s) I0930 13:48:08.161565 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (4m59.200259834s) I0930 13:48:10.402916 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (5m1.441613292s) I0930 13:48:12.631277 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (5m3.670076667s) I0930 13:48:14.865058 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (5m5.903410792s) I0930 13:48:17.095128 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (5m8.1339845s) I0930 13:48:19.348402 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (5m10.387256792s) I0930 13:48:21.583249 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (5m12.622047334s) I0930 13:48:23.865080 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (5m14.903891125s) I0930 13:48:26.111558 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (5m17.150402s) I0930 13:48:28.346557 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (5m19.385393375s) I0930 13:48:30.609260 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (5m21.648143625s) I0930 13:48:32.849510 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (5m23.8884275s) I0930 13:48:35.085113 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (5m26.124013834s) I0930 13:48:37.320957 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (5m28.359930792s) I0930 13:48:39.548899 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (5m30.587856125s) I0930 13:48:41.778829 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (5m32.817804167s) I0930 13:48:44.011286 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (5m35.050368917s) I0930 13:48:46.258309 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (5m37.297345167s) I0930 13:48:48.521235 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (5m39.560331209s) I0930 13:48:50.756385 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (5m41.795524167s) I0930 13:48:52.983231 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (5m44.0223885s) I0930 13:48:55.212551 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (5m46.251417667s) I0930 13:48:57.461756 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (5m48.500916875s) I0930 13:48:59.693280 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (5m50.732490959s) I0930 13:49:01.937918 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (5m52.977146084s) I0930 13:49:04.175989 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (5m55.215209875s) I0930 13:49:06.407158 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (5m57.446335209s) I0930 13:49:08.636445 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (5m59.6756515s) I0930 13:49:10.868008 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (6m1.907251s) I0930 13:49:13.106755 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (6m4.146041167s) I0930 13:49:15.344536 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (6m6.381165s) I0930 13:49:17.580387 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (6m8.614279084s) I0930 13:49:19.821579 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (6m10.853421375s) I0930 13:49:22.059890 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (6m13.089728375s) I0930 13:49:24.295038 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (6m15.32296075s) I0930 13:49:26.532664 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (6m17.55935525s) I0930 13:49:28.769738 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (6m19.795145375s) I0930 13:49:31.028421 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (6m22.050303042s) I0930 13:49:33.264812 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (6m24.288226375s) I0930 13:49:35.495344 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (6m26.517708292s) I0930 13:49:37.731726 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (6m28.753574959s) I0930 13:49:40.000151 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (6m31.021415667s) I0930 13:49:42.232305 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (6m33.253062417s) I0930 13:49:44.468252 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (6m35.488536917s) I0930 13:49:46.706485 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (6m37.726378375s) I0930 13:49:48.940505 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (6m39.960063375s) I0930 13:49:51.168933 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (6m42.188220709s) I0930 13:49:53.402394 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (6m44.421448667s) I0930 13:49:55.638015 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (6m46.656757709s) I0930 13:49:57.880001 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (6m48.898565334s) I0930 13:50:00.113781 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (6m51.132165875s) I0930 13:50:02.367282 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (6m53.38556825s) I0930 13:50:04.599337 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (6m55.617509s) I0930 13:50:06.832242 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (6m57.850302875s) I0930 13:50:09.062948 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (7m0.080924042s) I0930 13:50:11.306921 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (7m2.324899084s) I0930 13:50:13.552949 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (7m4.570776792s) I0930 13:50:15.788242 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (7m6.806065s) I0930 13:50:18.027644 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (7m9.04535325s) I0930 13:50:20.257684 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (7m11.2753995s) I0930 13:50:22.490574 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (7m13.508193292s) I0930 13:50:24.730980 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (7m15.748705959s) I0930 13:50:26.968842 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (7m17.986454917s) I0930 13:50:29.204776 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (7m20.222430709s) I0930 13:50:31.440992 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (7m22.458640792s) I0930 13:50:33.683695 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (7m24.701277417s) I0930 13:50:35.925096 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (7m26.942651084s) I0930 13:50:38.159283 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (7m29.176868334s) I0930 13:50:40.395881 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (7m31.413457042s) I0930 13:50:42.631507 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (7m33.64914325s) I0930 13:50:44.898019 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (7m35.915591375s) I0930 13:50:47.133778 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (7m38.151288125s) I0930 13:50:49.365444 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (7m40.383006334s) I0930 13:50:51.597132 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (7m42.61468625s) I0930 13:50:53.826596 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (7m44.844176625s) I0930 13:50:56.960596 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (7m47.97807675s) I0930 13:50:59.195016 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (7m50.212627542s) I0930 13:51:01.428760 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (7m52.446415834s) I0930 13:51:03.660576 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (7m54.6780875s) I0930 13:51:05.893984 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (7m56.911558459s) I0930 13:51:08.137437 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (7m59.155066042s) I0930 13:51:10.371135 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (8m1.388756292s) I0930 13:51:12.631185 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (8m3.648831792s) I0930 13:51:14.868353 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (8m5.885927625s) I0930 13:51:17.097066 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (8m8.114704375s) I0930 13:51:19.327951 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (8m10.345541167s) I0930 13:51:21.562292 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (8m12.580004042s) I0930 13:51:23.805789 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (8m14.823392209s) I0930 13:51:26.040547 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (8m17.058258334s) I0930 13:51:28.290171 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (8m19.307905084s) I0930 13:51:30.527123 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (8m21.54479675s) I0930 13:51:32.777100 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (8m23.794781s) I0930 13:51:35.011724 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (8m26.029472917s) I0930 13:51:37.264435 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (8m28.281919417s) I0930 13:51:39.501655 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (8m30.519377334s) I0930 13:51:41.734982 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (8m32.752693167s) I0930 13:51:43.970447 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (8m34.988222s) I0930 13:51:46.207406 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (8m37.225164167s) I0930 13:51:48.443122 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (8m39.460885292s) I0930 13:51:50.672649 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (8m41.690465709s) I0930 13:51:52.901519 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (8m43.9193335s) I0930 13:51:55.135564 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (8m46.153377959s) I0930 13:51:57.368505 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (8m48.386275667s) I0930 13:51:59.607393 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (8m50.62516975s) I0930 13:52:01.837863 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (8m52.855617125s) I0930 13:52:04.076559 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (8m55.094376417s) I0930 13:52:06.309791 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (8m57.327596625s) I0930 13:52:08.545519 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (8m59.563316584s) I0930 13:52:10.787037 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (9m1.804890292s) I0930 13:52:13.021157 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (9m4.038926s) I0930 13:52:15.251215 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (9m6.269041459s) I0930 13:52:17.545453 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (9m8.563326459s) I0930 13:52:19.789474 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (9m10.807333917s) I0930 13:52:22.027384 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (9m13.045226s) I0930 13:52:24.257946 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (9m15.275795667s) I0930 13:52:26.530538 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (9m17.548416334s) I0930 13:52:28.767476 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (9m19.78493275s) I0930 13:52:30.998785 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (9m22.016682334s) I0930 13:52:33.228890 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (9m24.246650625s) I0930 13:52:35.465969 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (9m26.483949792s) I0930 13:52:37.706339 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (9m28.72427225s) I0930 13:52:39.938277 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (9m30.956221125s) I0930 13:52:42.171682 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (9m33.189634209s) I0930 13:52:44.400417 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (9m35.418361625s) I0930 13:52:46.634981 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (9m37.65293375s) I0930 13:52:48.883791 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (9m39.901815709s) I0930 13:52:51.120037 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (9m42.138027084s) I0930 13:52:53.350867 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (9m44.36885375s) I0930 13:52:55.580069 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (9m46.598083292s) I0930 13:52:57.830430 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (9m48.848487459s) I0930 13:53:00.069675 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (9m51.087650792s) I0930 13:53:02.306188 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (9m53.324275125s) I0930 13:53:04.562032 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (9m55.580149292s) I0930 13:53:06.810444 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (9m57.828473917s) I0930 13:53:09.052305 67684 util.go:3890] Namespace csi-1613 found and status={Terminating [{NamespaceDeletionDiscoveryFailure False 2025-09-30 13:43:21 +0530 IST ResourcesDiscovered All resources successfully discovered} {NamespaceDeletionGroupVersionParsingFailure False 2025-09-30 13:43:21 +0530 IST ParsedGroupVersions All legacy kube types successfully parsed} {NamespaceDeletionContentFailure False 2025-09-30 13:43:25 +0530 IST ContentDeleted All content successfully deleted, may be waiting on finalization} {NamespaceContentRemaining True 2025-09-30 13:43:21 +0530 IST SomeResourcesRemain Some resources are remaining: cnsnodevmattachments.cns.vmware.com has 1 resource instances, persistentvolumeclaims. has 1 resource instances, zones.topology.tanzu.vmware.com has 1 resource instances} {NamespaceFinalizersRemaining True 2025-09-30 13:43:21 +0530 IST SomeFinalizersRemain Some content in the namespace has finalizers remaining: cns.vmware.com in 1 resource instances, cns.vmware.com/pvc-protection in 1 resource instances, tanzu-topology.vmware.com/zone in 1 resource instances}]} (10m0.070341375s) [FAILED] Expected success, but got an error: <*errors.errorString | 0x14000b88d50>: namespace csi-1613 still exists within 10m0s { s: "namespace csi-1613 still exists within 10m0s", } In [AfterEach] at: /Users/nayakb/vsphere-csi-driver/tests/e2e/restore_on_target_ds/restore_on_target_ds_vms_tests.go:153 @ 09/30/25 13:53:11.054 < Exit [AfterEach] [restore-on-target-ds-vms-p0] restore-on-target-ds-vms-p0 - /Users/nayakb/vsphere-csi-driver/tests/e2e/restore_on_target_ds/restore_on_target_ds_vms_tests.go:139 @ 09/30/25 13:53:11.055 (10m3.729s) > Enter [DeferCleanup (Each)] [restore-on-target-ds-vms-p0] restore-on-target-ds-vms-p0 - dump namespaces | framework.go:218 @ 09/30/25 13:53:11.055 STEP: dump namespace information after failure - /Users/nayakb/vsphere-csi-driver/vendor/k8s.io/kubernetes/test/e2e/framework/framework.go:297 @ 09/30/25 13:53:11.055 END STEP: dump namespace information after failure - /Users/nayakb/vsphere-csi-driver/vendor/k8s.io/kubernetes/test/e2e/framework/framework.go:297 @ 09/30/25 13:53:11.055 (0s) < Exit [DeferCleanup (Each)] [restore-on-target-ds-vms-p0] restore-on-target-ds-vms-p0 - dump namespaces | framework.go:218 @ 09/30/25 13:53:11.055 (0s) > Enter [DeferCleanup (Each)] [restore-on-target-ds-vms-p0] restore-on-target-ds-vms-p0 - tear down framework | framework.go:215 @ 09/30/25 13:53:11.055 < Exit [DeferCleanup (Each)] [restore-on-target-ds-vms-p0] restore-on-target-ds-vms-p0 - tear down framework | framework.go:215 @ 09/30/25 13:53:11.055 (0s) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/tests/e2e/restore_on_target_ds/restore_on_target_ds_tests.go b/tests/e2e/restore_on_target_ds/restore_on_target_ds_tests.go new file mode 100644 index 0000000000..3b276eb18b --- /dev/null +++ b/tests/e2e/restore_on_target_ds/restore_on_target_ds_tests.go @@ -0,0 +1,373 @@ +/* +Copyright 2025 The Kubernetes Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package restore_on_target_ds + +import ( + "context" + "time" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + + "sigs.k8s.io/vsphere-csi-driver/v3/tests/e2e/bootstrap" + "sigs.k8s.io/vsphere-csi-driver/v3/tests/e2e/config" + "sigs.k8s.io/vsphere-csi-driver/v3/tests/e2e/constants" + "sigs.k8s.io/vsphere-csi-driver/v3/tests/e2e/csisnapshot" + "sigs.k8s.io/vsphere-csi-driver/v3/tests/e2e/env" + + "sigs.k8s.io/vsphere-csi-driver/v3/tests/e2e/vcutil" + + corev1 "k8s.io/api/core/v1" + v1 "k8s.io/api/storage/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/kubernetes/test/e2e/framework" + fpv "k8s.io/kubernetes/test/e2e/framework/pv" + fss "k8s.io/kubernetes/test/e2e/framework/statefulset" + admissionapi "k8s.io/pod-security-admission/api" + "sigs.k8s.io/vsphere-csi-driver/v3/tests/e2e/k8testutil" +) + +var e2eTestConfig *config.E2eTestConfig + +var _ bool = ginkgo.Describe("[restore-on-target-ds-p0] restore-on-target-ds-p0", func() { + + f := framework.NewDefaultFramework("restore-on-target-ds") + f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged + f.SkipNamespaceCreation = true + var ( + client clientset.Interface + namespace string + storageclass *v1.StorageClass + restoreSc *v1.StorageClass + err error + vcRestSessionId string + statuscode int + storagePolicy string + restoreSp string + ) + + ginkgo.BeforeEach(func() { + e2eTestConfig = bootstrap.Bootstrap() + client = f.ClientSet + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // Read Env variable needed for the test suite + storagePolicy = env.GetAndExpectStringEnvVar(constants.EnvStoragePolicy) + restoreSp = env.GetAndExpectStringEnvVar(constants.EnvRestoreStoragePolicy) + + // Get the storageclass from storagepolicy + storageclass, err = client.StorageV1().StorageClasses().Get(ctx, storagePolicy, metav1.GetOptions{}) + if !apierrors.IsNotFound(err) { + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + } + + restoreSc, err = client.StorageV1().StorageClasses().Get(ctx, restoreSp, metav1.GetOptions{}) + if !apierrors.IsNotFound(err) { + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + } + + // reading vc session id + if vcRestSessionId == "" { + vcRestSessionId = k8testutil.CreateVcSession4RestApis(ctx, e2eTestConfig) + } + + storagePolicyId := vcutil.GetSpbmPolicyID(storagePolicy, e2eTestConfig) + targetPolicyId := vcutil.GetSpbmPolicyID(restoreSp, e2eTestConfig) + + // creating namespace with storagePolicy + if e2eTestConfig.TestInput.ClusterFlavor.SupervisorCluster { + namespace, statuscode, err = k8testutil.CreatetWcpNsWithZonesAndPolicies(e2eTestConfig, vcRestSessionId, + []string{storagePolicyId, targetPolicyId}, k8testutil.GetSvcId(vcRestSessionId, e2eTestConfig), + []string{}, "", "") + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(statuscode).To(gomega.Equal(204)) + } else { + labels_ns := map[string]string{} + labels_ns[admissionapi.EnforceLevelLabel] = string(admissionapi.LevelPrivileged) + labels_ns["e2e-framework"] = f.BaseName + gcNs, err := framework.CreateTestingNS(ctx, f.BaseName, client, labels_ns) + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "Error creating namespace on GC") + namespace = gcNs.Name + } + + //After NS creation need sometime to load usage CRs + time.Sleep(constants.PollTimeoutShort) + }) + + ginkgo.AfterEach(func() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + framework.Logf("Collecting supervisor PVC events before performing PV/PVC cleanup") + k8testutil.DumpSvcNsEventsOnTestFailure(client, namespace) + eventList, err := client.CoreV1().Events(namespace).List(ctx, metav1.ListOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + for _, item := range eventList.Items { + framework.Logf("%q", item.Message) + } + + // TODO Delete namespace + + }) + + // TC-17 Failing due to PR-3576259 + ginkgo.It("Online & offline volume expansion of restored volume", ginkgo.Label( + constants.P0, constants.Vc901), func() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + framework.Logf("Starting test: Online & offline volume expansion of restored volume") + + // Create PVC + pvclaim, pvList := k8testutil.CreateAndValidatePvc(ctx, client, namespace, storageclass) + + // Create pod + _, err = k8testutil.CreatePod(ctx, e2eTestConfig, client, namespace, nil, []*corev1.PersistentVolumeClaim{pvclaim}, false, + constants.ExecRWXCommandPod1) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + vs, _ := csisnapshot.CreateVolumeSnapshot(ctx, e2eTestConfig, namespace, pvclaim, pvList, constants.DiskSize) + + ginkgo.By("Restore sanpshots to create new volumes") + restoredPvc, _, _ := csisnapshot.VerifyVolumeRestoreOperation(ctx, e2eTestConfig, client, namespace, restoreSc, + vs, constants.DiskSize, true) + + // Run online expansion + currentPvcSize := restoredPvc.Spec.Resources.Requests[corev1.ResourceStorage] + newSize := currentPvcSize.DeepCopy() + newSize.Add(resource.MustParse("4Gi")) + framework.Logf("currentPvcSize %v, newSize %v", currentPvcSize, newSize) + _, err = k8testutil.ExpandPVCSize(restoredPvc, newSize, client) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + ginkgo.By("Waiting for file system resize to finish") + restoredPvc, err = k8testutil.WaitForFSResize(restoredPvc, client) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + pvcConditions := restoredPvc.Status.Conditions + k8testutil.ExpectEqual(len(pvcConditions), 0, "pvc should not have conditions") + + // Restore again for offline expansion + ginkgo.By("Restore sanpshots to create new volumes for offline expansion") + restoredPvc2, _, _ := csisnapshot.VerifyVolumeRestoreOperation(ctx, e2eTestConfig, client, namespace, restoreSc, + vs, constants.DiskSize, false) + + currentPvcSize = restoredPvc2.Spec.Resources.Requests[corev1.ResourceStorage] + newSize = currentPvcSize.DeepCopy() + newSize.Add(resource.MustParse("1Gi")) + framework.Logf("currentPvcSize %v, newSize %v", currentPvcSize, newSize) + restoredPvc2, err = k8testutil.ExpandPVCSize(restoredPvc, newSize, client) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(restoredPvc2).NotTo(gomega.BeNil()) + b, err := k8testutil.VerifyPvcRequestedSizeUpdateInSupervisorWithWait(restoredPvc2.Name, newSize) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(b).To(gomega.BeTrue()) + + framework.Logf("Ending test: Online & offline volume expansion of restored volume") + + }) + + // TC-16 + ginkgo.It("Restore with different size", ginkgo.Label( + constants.P0, constants.Vc901), func() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + framework.Logf("Starting test: Restore with different size") + + // Create a PVC on sc-1 and attach pod to it + volumeMap := k8testutil.CreateMultiplePvcPod(ctx, e2eTestConfig, client, namespace, storageclass, 1, true, false) + + // Create a snapshot of the volume. + for pvclaim, pvList := range volumeMap { + volumeSnapshot, _ := csisnapshot.CreateVolumeSnapshot(ctx, e2eTestConfig, namespace, pvclaim, pvList, constants.DiskSize) + + // Restore it on different SC + ginkgo.By("Restore sanpshots to create new volumes") + pvcSpec := csisnapshot.GetPersistentVolumeClaimSpecWithDatasource(namespace, constants.DiskSize1GB, restoreSc, nil, + corev1.ReadWriteOnce, volumeSnapshot.Name, constants.Snapshotapigroup) + + restoredPvc, err := fpv.CreatePVC(ctx, client, namespace, pvcSpec) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + _, err = fpv.WaitForPVClaimBoundPhase(ctx, client, + []*corev1.PersistentVolumeClaim{restoredPvc}, framework.ClaimProvisionTimeout) + gomega.Expect(err).To(gomega.HaveOccurred()) + } + + framework.Logf("Ending test: Restore with different size") + }) + + // TC -15 + ginkgo.It("Create a snapshot of the restored PVC", ginkgo.Label( + constants.P0, constants.Vc901), func() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + framework.Logf("Starting test:Create a snapshot of the restored PVC") + + // Create a PVC on sc-1 and attach pod to it + volumeMap := k8testutil.CreateMultiplePvcPod(ctx, e2eTestConfig, client, namespace, storageclass, 1, true, false) + + // Create a snapshot of the volume. + for pvclaim, pvList := range volumeMap { + volumeSnapshot, _ := csisnapshot.CreateVolumeSnapshot(ctx, e2eTestConfig, namespace, pvclaim, pvList, constants.DiskSize) + + // Restore it on different SC + ginkgo.By("Restore sanpshots to create new volumes") + restoredPvc, restoredPvList, _ := csisnapshot.VerifyVolumeRestoreOperation(ctx, e2eTestConfig, client, namespace, restoreSc, + volumeSnapshot, constants.DiskSize, true) + + // create snapshot of volume created in #4 + vsRestoredPvc, _ := csisnapshot.CreateVolumeSnapshot(ctx, e2eTestConfig, namespace, restoredPvc, restoredPvList, constants.DiskSize) + + // Restore it with different sc + ginkgo.By("Restore sanpshots to create new volumes") + _, _, _ = csisnapshot.VerifyVolumeRestoreOperation(ctx, e2eTestConfig, client, namespace, storageclass, + vsRestoredPvc, constants.DiskSize, false) + } + + framework.Logf("Ending test: Create a snapshot of the restored PVC") + }) + + // TC -14 + ginkgo.It("Restore with a different policy on Deployment pods", ginkgo.Label( + constants.P0, constants.Vc901), func() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + framework.Logf("Starting test: Restore with a different policy on Deployment ") + + // Create a PVC on sc-1 and attach pod to it + volumeMap := k8testutil.CreateMultiplePvcPod(ctx, e2eTestConfig, client, namespace, storageclass, 1, false, true) + + // Create a snapshot of the volume. + for pvclaim, pvList := range volumeMap { + volumeSnapshot, _ := csisnapshot.CreateVolumeSnapshot(ctx, e2eTestConfig, namespace, pvclaim, pvList, constants.DiskSize) + + // Restore it on different SC + ginkgo.By("Restore sanpshots to create new volumes") + _, _, _ = csisnapshot.VerifyVolumeRestoreOperation(ctx, e2eTestConfig, client, namespace, restoreSc, + volumeSnapshot, constants.DiskSize, true) + } + + framework.Logf("Ending test: Restore with a different policy on Deployment ") + }) + + // TC - 13 + ginkgo.It("Restore with a different policy on statefulset ", ginkgo.Label( + constants.P0, constants.Vc901), func() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + framework.Logf("Starting test: Restore with a different policy on statefulset") + + // Create a statefulset with replica=3 + ginkgo.By("Creating statefulset") + var replicas int32 = 3 + ginkgo.By("Creating service") + _ = k8testutil.CreateService(namespace, client) + statefulset := k8testutil.CreateCustomisedStatefulSets(ctx, client, e2eTestConfig.TestInput, namespace, true, replicas, false, nil, + false, true, "", "", storageclass, storageclass.Name) + + // List the STS Pvcs + stsPod, err := fss.GetPodList(ctx, client, statefulset) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + volumeMap := map[*corev1.PersistentVolumeClaim]*corev1.PersistentVolume{} + for _, pod := range stsPod.Items { + for _, volumeSpec := range pod.Spec.Volumes { + if volumeSpec.PersistentVolumeClaim != nil { + svPvcName := volumeSpec.PersistentVolumeClaim.ClaimName + pv := k8testutil.GetPvFromClaim(client, namespace, svPvcName) + // Get SVC PVC + svcPVC, err := client.CoreV1().PersistentVolumeClaims(namespace).Get(ctx, svPvcName, metav1.GetOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + volumeMap[svcPVC] = pv + } + } + } + + // Create a snapshot for a PVC + for pvclaim, pv := range volumeMap { + volumeSnapshot, _ := csisnapshot.CreateVolumeSnapshot(ctx, e2eTestConfig, namespace, pvclaim, []*corev1.PersistentVolume{pv}, constants.DiskSize1GB) + + // Restore it on different SC + ginkgo.By("Restore sanpshots to create new volumes") + _, _, _ = csisnapshot.VerifyVolumeRestoreOperation(ctx, e2eTestConfig, client, namespace, restoreSc, + volumeSnapshot, constants.DiskSize1GB, true) + + break + } + + framework.Logf("Ending test: Restore with a different policy on statefulset") + }) + + // TC - 9 + ginkgo.It("WFFC - Create a PVC from a snapshot on a different datastore", ginkgo.Label( + constants.P0, constants.Vc901), func() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + framework.Logf("Starting test: WFFC - Create a PVC from a snapshot on a different datastore") + + storagePolicyWffc := storagePolicy + "-latebinding" + restoreSpwffc := restoreSp + "-latebinding" + // Get the storageclass from storagepolicy + storageclassWffc, err := client.StorageV1().StorageClasses().Get(ctx, storagePolicyWffc, metav1.GetOptions{}) + if !apierrors.IsNotFound(err) { + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + } + restoreScWffc, err := client.StorageV1().StorageClasses().Get(ctx, restoreSpwffc, metav1.GetOptions{}) + if !apierrors.IsNotFound(err) { + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + } + + // Create PVC + labelsMap := make(map[string]string) + labelsMap["app"] = "test" + pvc, err := k8testutil.CreatePVC(ctx, client, namespace, labelsMap, "", storageclassWffc, "") + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + // create pod + ginkgo.By("Create Pod to attach to Pvc-2") + _, err = k8testutil.CreatePod(ctx, e2eTestConfig, client, namespace, nil, []*corev1.PersistentVolumeClaim{pvc}, false, + constants.ExecRWXCommandPod1) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + pvs, err := fpv.WaitForPVClaimBoundPhase(ctx, client, []*corev1.PersistentVolumeClaim{pvc}, constants.PollTimeout) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + volumeSnapshot, _ := csisnapshot.CreateVolumeSnapshot(ctx, e2eTestConfig, namespace, pvc, pvs, constants.DiskSize) + + // Restore it on different SC-wffc + ginkgo.By("Restore sanpshots to create new volumes") + pvcSpec := csisnapshot.GetPersistentVolumeClaimSpecWithDatasource(namespace, constants.DiskSize, restoreScWffc, nil, + corev1.ReadWriteOnce, volumeSnapshot.Name, constants.Snapshotapigroup) + + restoredPvc, err := fpv.CreatePVC(ctx, client, namespace, pvcSpec) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + ginkgo.By("Create Pod to attach to Pvc-2") + _, err = k8testutil.CreatePod(ctx, e2eTestConfig, client, namespace, nil, []*corev1.PersistentVolumeClaim{restoredPvc}, false, + constants.ExecRWXCommandPod1) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + _, err = fpv.WaitForPVClaimBoundPhase(ctx, client, []*corev1.PersistentVolumeClaim{restoredPvc}, constants.PollTimeout) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + framework.Logf("Ending test:WFFC - Create a PVC from a snapshot on a different datastore") + + }) + +}) diff --git a/tests/e2e/restore_on_target_ds/restore_on_target_ds_vms_tests.go b/tests/e2e/restore_on_target_ds/restore_on_target_ds_vms_tests.go new file mode 100644 index 0000000000..0e2dbf9703 --- /dev/null +++ b/tests/e2e/restore_on_target_ds/restore_on_target_ds_vms_tests.go @@ -0,0 +1,396 @@ +/* +Copyright 2025 The Kubernetes Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package restore_on_target_ds + +import ( + "context" + "fmt" + "math/rand" + "os" + "os/exec" + "strings" + "time" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + + "sigs.k8s.io/vsphere-csi-driver/v3/tests/e2e/bootstrap" + "sigs.k8s.io/vsphere-csi-driver/v3/tests/e2e/constants" + "sigs.k8s.io/vsphere-csi-driver/v3/tests/e2e/csisnapshot" + "sigs.k8s.io/vsphere-csi-driver/v3/tests/e2e/env" + "sigs.k8s.io/vsphere-csi-driver/v3/tests/e2e/vmservice_vm" + + "sigs.k8s.io/vsphere-csi-driver/v3/tests/e2e/vcutil" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha1" + corev1 "k8s.io/api/core/v1" + v1 "k8s.io/api/storage/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/kubernetes/test/e2e/framework" + admissionapi "k8s.io/pod-security-admission/api" + ctlrclient "sigs.k8s.io/controller-runtime/pkg/client" + cnsop "sigs.k8s.io/vsphere-csi-driver/v3/pkg/apis/cnsoperator" + "sigs.k8s.io/vsphere-csi-driver/v3/tests/e2e/k8testutil" +) + +var _ bool = ginkgo.Describe("[restore-on-target-ds-vms-p0] restore-on-target-ds-vms-p0", func() { + + f := framework.NewDefaultFramework("restore-on-target-ds-vms") + f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged + f.SkipNamespaceCreation = true + var ( + client clientset.Interface + namespace string + storageclass *v1.StorageClass + restoreSc *v1.StorageClass + err error + vcRestSessionId string + statuscode int + storagePolicy string + restoreSp string + contentLibId string + vmClass string + vmopC ctlrclient.Client + cnsopC ctlrclient.Client + ) + + ginkgo.BeforeEach(func() { + e2eTestConfig = bootstrap.Bootstrap() + client = f.ClientSet + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // Read Env variable needed for the test suite + storagePolicy = env.GetAndExpectStringEnvVar(constants.EnvStoragePolicy) + restoreSp = env.GetAndExpectStringEnvVar(constants.EnvRestoreStoragePolicy) + + // Get the storageclass from storagepolicy + storageclass, err = client.StorageV1().StorageClasses().Get(ctx, storagePolicy, metav1.GetOptions{}) + if !apierrors.IsNotFound(err) { + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + } + + restoreSc, err = client.StorageV1().StorageClasses().Get(ctx, restoreSp, metav1.GetOptions{}) + if !apierrors.IsNotFound(err) { + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + } + + // reading vc session id + if vcRestSessionId == "" { + vcRestSessionId = k8testutil.CreateVcSession4RestApis(ctx, e2eTestConfig) + } + + storagePolicyId := vcutil.GetSpbmPolicyID(storagePolicy, e2eTestConfig) + + // fetch shared vsphere datatsore url + datastoreURL := env.GetAndExpectStringEnvVar(constants.EnvSharedDatastoreURL) + dsRef := vcutil.GetDsMoRefFromURL(ctx, e2eTestConfig, datastoreURL) + framework.Logf("dsmoId: %v", dsRef.Value) + + // read or create content library if it is empty + if contentLibId == "" { + contentLibId, err = vmservice_vm.CreateAndOrGetContentlibId4Url(e2eTestConfig, vcRestSessionId, env.GetAndExpectStringEnvVar(constants.EnvContentLibraryUrl), + dsRef.Value) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + } + + // get or set vm class required for VM creation + vmClass = os.Getenv(constants.EnvVMClass) + if vmClass == "" { + vmClass = constants.VmClassBestEffortSmall + } + /* Sets up a Kubernetes client with a custom scheme, adds the vmopv1 API types to the scheme, + and ensures that the client is properly initialized without errors */ + vmopScheme := runtime.NewScheme() + gomega.Expect(vmopv1.AddToScheme(vmopScheme)).Should(gomega.Succeed()) + vmopC, err = ctlrclient.New(f.ClientConfig(), ctlrclient.Options{Scheme: vmopScheme}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + cnsOpScheme := runtime.NewScheme() + gomega.Expect(cnsop.AddToScheme(cnsOpScheme)).Should(gomega.Succeed()) + cnsopC, err = ctlrclient.New(f.ClientConfig(), ctlrclient.Options{Scheme: cnsOpScheme}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + // creating namespace with storagePolicy + namespace, statuscode, err = k8testutil.CreatetWcpNsWithZonesAndPolicies(e2eTestConfig, vcRestSessionId, + []string{storagePolicyId}, k8testutil.GetSvcId(vcRestSessionId, e2eTestConfig), + []string{}, vmClass, contentLibId) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(statuscode).To(gomega.Equal(204)) + + //After NS creation need sometime to load usage CRs + time.Sleep(constants.PollTimeoutShort) + }) + + ginkgo.AfterEach(func() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + framework.Logf("Collecting supervisor PVC events before performing PV/PVC cleanup") + k8testutil.DumpSvcNsEventsOnTestFailure(client, namespace) + eventList, err := client.CoreV1().Events(namespace).List(ctx, metav1.ListOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + for _, item := range eventList.Items { + framework.Logf("%q", item.Message) + } + + // Delete namespace + k8testutil.DelTestWcpNs(e2eTestConfig, vcRestSessionId, namespace) + gomega.Expect(k8testutil.WaitForNamespaceToGetDeleted(ctx, client, namespace, constants.Poll, constants.PollTimeout)).To(gomega.Succeed()) + + }) + + // TC-12 + ginkgo.It("Attach VM to restored PVC which already attached to original PVC", ginkgo.Label( + constants.P0, constants.Vc901), func() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + framework.Logf("Starting test: Attach VM to restored PVC which already attached to original PVC") + + // Create a PVC on sc-1 and attach them to VM + pvclaim, pvList := k8testutil.CreateAndValidatePvc(ctx, client, namespace, storageclass) + + ginkgo.By("Create vm service vm") + _, vm, _, err := vmservice_vm.CreateVmServiceVm(ctx, client, vmopC, cnsopC, namespace, + []*corev1.PersistentVolumeClaim{pvclaim}, vmClass, storageclass.Name, true) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + ginkgo.By("Wait for VM to come up and get an IP") + _, err = vmservice_vm.WaitNgetVmsvcVmIp(ctx, vmopC, namespace, vm.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + ginkgo.By("Wait and verify PVCs are attached to the VM") + gomega.Expect(vmservice_vm.WaitNverifyPvcsAreAttachedToVmsvcVm(ctx, vmopC, cnsopC, vm, + []*corev1.PersistentVolumeClaim{pvclaim})).NotTo(gomega.HaveOccurred()) + + // Create a snapshot of both the volume. + volumeSnapshot, _ := csisnapshot.CreateVolumeSnapshot(ctx, e2eTestConfig, namespace, pvclaim, pvList, constants.DiskSize) + + // Restore it on different SC + ginkgo.By("Restore sanpshots to create new volumes") + restoredPvc, _, _ := csisnapshot.VerifyVolumeRestoreOperation(ctx, e2eTestConfig, client, namespace, restoreSc, + volumeSnapshot, constants.DiskSize, false) + + // attach restored pvc to original VM + ginkgo.By("Attach restored PVC to original vm") + vm, err = vmservice_vm.GetVmsvcVM(ctx, vmopC, vm.Namespace, vm.Name) // refresh vm info + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + vm.Spec.Volumes = append(vm.Spec.Volumes, vmopv1.VirtualMachineVolume{Name: restoredPvc.Name, + PersistentVolumeClaim: &vmopv1.PersistentVolumeClaimVolumeSource{ + PersistentVolumeClaimVolumeSource: corev1.PersistentVolumeClaimVolumeSource{ClaimName: restoredPvc.Name}, + }}) + err = vmopC.Update(ctx, vm) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + // TODO : Run volume expansion on both the restored PVCs + + framework.Logf("Ending test: Attach VM to restored PVC which already attached to original PVC") + + }) + + // TC -11 + ginkgo.It("Attach VM to restored PVC with detaching original PVC", ginkgo.Label( + constants.P0, constants.Vc901), func() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + framework.Logf("Starting test: Attach VM to restored PVC with detaching original PVC") + + // Create a PVC on sc-1 and attach them to VM + pvclaim, pvList := k8testutil.CreateAndValidatePvc(ctx, client, namespace, storageclass) + + ginkgo.By("Create vm service vm") + _, vm, _, err := vmservice_vm.CreateVmServiceVm(ctx, client, vmopC, cnsopC, namespace, + []*corev1.PersistentVolumeClaim{pvclaim}, vmClass, storageclass.Name, true) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + ginkgo.By("Wait for VM to come up and get an IP") + _, err = vmservice_vm.WaitNgetVmsvcVmIp(ctx, vmopC, namespace, vm.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + ginkgo.By("Wait and verify PVCs are attached to the VM") + gomega.Expect(vmservice_vm.WaitNverifyPvcsAreAttachedToVmsvcVm(ctx, vmopC, cnsopC, vm, + []*corev1.PersistentVolumeClaim{pvclaim})).NotTo(gomega.HaveOccurred()) + + // Create a snapshot of both the volume. + volumeSnapshot, _ := csisnapshot.CreateVolumeSnapshot(ctx, e2eTestConfig, namespace, pvclaim, pvList, constants.DiskSize) + + // Restore it on different SC + ginkgo.By("Restore sanpshots to create new volumes") + restoredPvc, _, _ := csisnapshot.VerifyVolumeRestoreOperation(ctx, e2eTestConfig, client, namespace, restoreSc, + volumeSnapshot, constants.DiskSize, false) + + // detach original volume from VM + ginkgo.By("Edit vm spec and remove volume attached to it") + vm, err = vmservice_vm.GetVmsvcVM(ctx, vmopC, vm.Namespace, vm.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + vm.Spec.Volumes = []vmopv1.VirtualMachineVolume{} + err = vmopC.Update(ctx, vm) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + // Attach restored volume to vm + vm.Spec.Volumes = append(vm.Spec.Volumes, vmopv1.VirtualMachineVolume{Name: restoredPvc.Name, + PersistentVolumeClaim: &vmopv1.PersistentVolumeClaimVolumeSource{ + PersistentVolumeClaimVolumeSource: corev1.PersistentVolumeClaimVolumeSource{ClaimName: restoredPvc.Name}, + }}) + err = vmopC.Update(ctx, vm) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + vm, err = vmservice_vm.GetVmsvcVM(ctx, vmopC, vm.Namespace, vm.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + ginkgo.By("Wait and verify restored PVC is attached to vm") + gomega.Expect(vmservice_vm.WaitNverifyPvcsAreAttachedToVmsvcVm(ctx, vmopC, cnsopC, vm, + []*corev1.PersistentVolumeClaim{restoredPvc})).To(gomega.Succeed()) + + // TODO : Run volume expansion on both the restored PVCs + + framework.Logf("Ending test: Attach VM to restored PVC with detaching original PVC") + + }) + + // TC -10 + ginkgo.It("Create a PVC from a snapshot on a different datastore, verify the data integrity", ginkgo.Label( + constants.P0, constants.Vc901), func() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + framework.Logf("Starting test: Create a PVC from a snapshot on a different datastore, verify the data integrity") + + // Create a PVC on sc-1 and attach them to VM + pvclaim, pvList := k8testutil.CreateAndValidatePvc(ctx, client, namespace, storageclass) + + ginkgo.By("Create vm service vm") + _, vm, _, err := vmservice_vm.CreateVmServiceVm(ctx, client, vmopC, cnsopC, namespace, + []*corev1.PersistentVolumeClaim{pvclaim}, vmClass, storageclass.Name, true) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + ginkgo.By("Wait for VM to come up and get an IP") + _, err = vmservice_vm.WaitNgetVmsvcVmIp(ctx, vmopC, namespace, vm.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + ginkgo.By("Wait and verify PVCs are attached to the VM") + gomega.Expect(vmservice_vm.WaitNverifyPvcsAreAttachedToVmsvcVm(ctx, vmopC, cnsopC, vm, + []*corev1.PersistentVolumeClaim{pvclaim})).NotTo(gomega.HaveOccurred()) + + vmIp, err := vmservice_vm.WaitNgetVmsvcVmIp(ctx, vmopC, namespace, vm.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + volFolder := vmservice_vm.FormatNVerifyPvcIsAccessible(vm.Status.Volumes[0].DiskUuid, 1, vmIp) + + // Write data to VM + ginkgo.By("write some data to a file in PVC from VM") + rand.New(rand.NewSource(time.Now().Unix())) + testdataFile := fmt.Sprintf("/tmp/testdata_%v_%v", time.Now().Unix(), rand.Intn(1000)) + framework.Logf("Creating a 100mb test data file %v", testdataFile) + op, err := exec.Command( + "bash", "-c", "dd if=/dev/urandom bs=1M count=1 | tr -dc 'a-zA-Z0-9' >"+testdataFile).Output() + // using 'tr' to filter out non-alphanumeric characters + fmt.Println(op) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + op, err = exec.Command("md5sum", testdataFile).Output() + fmt.Println("md5sum", string(op[:])) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + op, err = exec.Command("ls", "-l", testdataFile).Output() + fmt.Println(string(op[:])) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + defer func() { + op, err = exec.Command("rm", "-f", testdataFile).Output() + fmt.Println(op) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }() + framework.Logf("Copying test data file to VM") + vmservice_vm.CopyFileToVm(vmIp, testdataFile, volFolder+"/f1") + + _ = vmservice_vm.ExecSshOnVmThroughGatewayVm(vmIp, + []string{"ls -l " + volFolder + "/f1", "md5sum " + volFolder + "/f1", "sync"}) + + // Create a snapshot of both the volume. + volumeSnapshot, _ := csisnapshot.CreateVolumeSnapshot(ctx, e2eTestConfig, namespace, pvclaim, pvList, constants.DiskSize) + + // Restore it on different SC + ginkgo.By("Restore sanpshots to create new volumes") + restoredPvc, _, _ := csisnapshot.VerifyVolumeRestoreOperation(ctx, e2eTestConfig, client, namespace, restoreSc, + volumeSnapshot, constants.DiskSize, false) + + // Attach the restored volume to a VM + _, vm2, _, err := vmservice_vm.CreateVmServiceVm(ctx, client, vmopC, cnsopC, namespace, + []*corev1.PersistentVolumeClaim{restoredPvc}, vmClass, restoreSc.Name, false) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + ginkgo.By("Wait for VM to come up and get an IP") + _, err = vmservice_vm.WaitNgetVmsvcVmIp(ctx, vmopC, namespace, vm2.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + ginkgo.By("Wait and verify PVCs are attached to the VM") + gomega.Expect(vmservice_vm.WaitNverifyPvcsAreAttachedToVmsvcVm(ctx, vmopC, cnsopC, vm2, + []*corev1.PersistentVolumeClaim{restoredPvc})).NotTo(gomega.HaveOccurred()) + + // Verify data exists + ginkgo.By("verify data in restored PVC from vm2") + framework.Logf("Mounting the volume") + vmIp2, err := vmservice_vm.WaitNgetVmsvcVmIp(ctx, vmopC, namespace, vm2.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + volFolder = vmservice_vm.FormatNVerifyPvcIsAccessible(vm2.Status.Volumes[0].DiskUuid, 1, vmIp2) + vmFileData := fmt.Sprintf("/tmp/vmdata_%v_%v", time.Now().Unix(), rand.Intn(1000)) + _ = vmservice_vm.ExecSshOnVmThroughGatewayVm(vmIp2, []string{"md5sum " + volFolder + "/f1"}) + framework.Logf("Fetching file from the VM") + vmservice_vm.CopyFileFromVm(vmIp2, volFolder+"/f1", vmFileData) + defer func() { + c := []string{"rm", "-f", vmFileData} + op, err = exec.Command(c[0], c[1:]...).Output() + framework.Logf("Command: %v, output: %v", c, op) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }() + framework.Logf("Comparing file fetched from the VM with test data file") + c := []string{"md5sum", testdataFile, vmFileData} + op, err = exec.Command(c[0], c[1:]...).Output() + framework.Logf("Command: %v, output: %v", c, op) + lines := strings.Split(string(op[:]), "\n") + gomega.Expect(strings.Fields(lines[0])[0]).To(gomega.Equal(strings.Fields(lines[1])[0])) + + // Write new data on the restored volume + ginkgo.By("write some data to a file in PVC from VM") + rand.New(rand.NewSource(time.Now().Unix())) + testdata2File := fmt.Sprintf("/tmp/testdata2_%v_%v", time.Now().Unix(), rand.Intn(1000)) + framework.Logf("Creating a 100mb test data file %v", testdata2File) + op, err = exec.Command( + "bash", "-c", "dd if=/dev/urandom bs=1M count=1 | tr -dc 'a-zA-Z0-9' >"+testdata2File).Output() + // using 'tr' to filter out non-alphanumeric characters + fmt.Println(op) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + op, err = exec.Command("md5sum", testdataFile).Output() + fmt.Println("md5sum", string(op[:])) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + op, err = exec.Command("ls", "-l", testdataFile).Output() + fmt.Println(string(op[:])) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + defer func() { + op, err = exec.Command("rm", "-f", testdataFile).Output() + fmt.Println(op) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }() + framework.Logf("Copying test data file to VM") + vmservice_vm.CopyFileToVm(vmIp, testdataFile, volFolder+"/f1") + + _ = vmservice_vm.ExecSshOnVmThroughGatewayVm(vmIp, + []string{"ls -l " + volFolder + "/f1", "md5sum " + volFolder + "/f1", "sync"}) + + framework.Logf("Ending test: Create a PVC from a snapshot on a different datastore, verify the data integrity") + + }) + +}) diff --git a/tests/e2e/restore_on_target_ds/suite_test.go b/tests/e2e/restore_on_target_ds/suite_test.go new file mode 100644 index 0000000000..83eed8de85 --- /dev/null +++ b/tests/e2e/restore_on_target_ds/suite_test.go @@ -0,0 +1,50 @@ +/* +Copyright 2025 The Kubernetes Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package restore_on_target_ds + +import ( + "flag" + "os" + "strings" + "testing" + + ginkgo "github.com/onsi/ginkgo/v2" + gomega "github.com/onsi/gomega" + "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/framework/config" + _ "k8s.io/kubernetes/test/e2e/framework/debug/init" + "sigs.k8s.io/vsphere-csi-driver/v3/tests/e2e/constants" +) + +func init() { + framework.AfterReadingAllFlags(&framework.TestContext) +} + +func TestE2E(t *testing.T) { + handleFlags() + gomega.RegisterFailHandler(ginkgo.Fail) + _, reporterConfig := ginkgo.GinkgoConfiguration() + reporterConfig.JUnitReport = "junit.xml" + ginkgo.RunSpecs(t, "End-to-End-Creating-Disk-From-Snapshot-On-Target-Datastore", reporterConfig) +} + +func handleFlags() { + config.CopyFlags(config.Flags, flag.CommandLine) + framework.RegisterCommonFlags(flag.CommandLine) + framework.TestContext.KubeConfig = os.Getenv(constants.KubeconfigEnvVar) + mydir, err := os.Getwd() + framework.ExpectNoError(err) + framework.TestContext.RepoRoot = strings.ReplaceAll(mydir, "/tests/e2e", "") + flag.Parse() +} diff --git a/tests/e2e/vcutil/vc_util.go b/tests/e2e/vcutil/vc_util.go index acf7bbf706..e393f084ae 100644 --- a/tests/e2e/vcutil/vc_util.go +++ b/tests/e2e/vcutil/vc_util.go @@ -330,7 +330,7 @@ func QueryCNSVolumeSnapshotWithResult(vs *config.E2eTestConfig, fcdID string, snapshotId string) (*cnstypes.CnsSnapshotQueryResult, error) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - + framework.Logf("fcdID : %s ; snapshotIds : %s", fcdID, snapshotId) var snapshotSpec []cnstypes.CnsSnapshotQuerySpec snapshotSpec = append(snapshotSpec, cnstypes.CnsSnapshotQuerySpec{ VolumeId: cnstypes.CnsVolumeId{ diff --git a/tests/e2e/vmservice_vm/rwx_vmsvc_utils.go b/tests/e2e/vmservice_vm/rwx_vmsvc_utils.go index 14596238e3..de8d05173c 100644 --- a/tests/e2e/vmservice_vm/rwx_vmsvc_utils.go +++ b/tests/e2e/vmservice_vm/rwx_vmsvc_utils.go @@ -71,7 +71,7 @@ func FetchNFSAccessPointFromCnsFileAccessConfigCRD(ctx context.Context, restConf // Helper function to run SSH and log result func RunSSHFromVmServiceVmAndLog(vmIP, cmd string) error { - output := execSshOnVmThroughGatewayVm(vmIP, []string{cmd}) + output := ExecSshOnVmThroughGatewayVm(vmIP, []string{cmd}) if output[0].Stderr != "" { return fmt.Errorf("command failed with error: %s", output[0].Stderr) } else { @@ -125,7 +125,7 @@ func MountRWXVolumeAndVerifyIO(vmIPs []string, nfsAccessPoint string, testDir st for _, writerVM := range vmIPs { fileName := fmt.Sprintf("%s-%s.txt", filePrefix, strings.ReplaceAll(writerVM, ".", "-")) readCmd := fmt.Sprintf("cat /mnt/nfs/%s/%s", testDir, fileName) - output := execSshOnVmThroughGatewayVm(readerVM, []string{readCmd}) + output := ExecSshOnVmThroughGatewayVm(readerVM, []string{readCmd}) if !strings.Contains(output[0].Stdout, writerVM) { return fmt.Errorf("VM %s failed to read file written by VM %s", readerVM, writerVM) } else { diff --git a/tests/e2e/vmservice_vm/vmservice_utils.go b/tests/e2e/vmservice_vm/vmservice_utils.go index fbf53ab725..246a167bc4 100644 --- a/tests/e2e/vmservice_vm/vmservice_utils.go +++ b/tests/e2e/vmservice_vm/vmservice_utils.go @@ -693,20 +693,20 @@ func FormatNVerifyPvcIsAccessible(diskUuid string, mountIndex int, vmIp string) fmt.Println("Checking disk path:", p) // List the available disks - results := execSshOnVmThroughGatewayVm(vmIp, []string{ + results := ExecSshOnVmThroughGatewayVm(vmIp, []string{ "ls -l /dev/disk/by-id/", }) fmt.Println("Disk list results:", results) // Check if the desired disk exists - diskCheckResults := execSshOnVmThroughGatewayVm(vmIp, []string{ + diskCheckResults := ExecSshOnVmThroughGatewayVm(vmIp, []string{ "ls -l " + p, }) // If the disk is not found, try rescanning SCSI devices if strings.Contains(diskCheckResults[0].Stderr, "No such file or directory") { fmt.Printf("Disk %s not found. Rescanning SCSI devices.\n", p) - rescanResults := execSshOnVmThroughGatewayVm(vmIp, []string{ + rescanResults := ExecSshOnVmThroughGatewayVm(vmIp, []string{ "echo '- - -' | sudo tee /sys/class/scsi_host/host*/scan", "ls -l /dev/disk/by-id/", "ls -l " + p, @@ -714,7 +714,7 @@ func FormatNVerifyPvcIsAccessible(diskUuid string, mountIndex int, vmIp string) fmt.Println("Rescan results:", rescanResults) // Check again if the disk is available after rescanning - diskCheckResults = execSshOnVmThroughGatewayVm(vmIp, []string{ + diskCheckResults = ExecSshOnVmThroughGatewayVm(vmIp, []string{ "ls -l " + p, }) } @@ -742,7 +742,7 @@ func FormatNVerifyPvcIsAccessible(diskUuid string, mountIndex int, vmIp string) unmountCommands := []string{ fmt.Sprintf("sudo umount %s* || true", dev), } - res := execSshOnVmThroughGatewayVm(vmIp, unmountCommands) + res := ExecSshOnVmThroughGatewayVm(vmIp, unmountCommands) fmt.Println("Unmount Results:", res) // Partition and format the disk @@ -752,7 +752,7 @@ func FormatNVerifyPvcIsAccessible(diskUuid string, mountIndex int, vmIp string) "lsblk -l", fmt.Sprintf("sudo mkfs.ext4 %s", partitionDev), } - res = execSshOnVmThroughGatewayVm(vmIp, partitionCommands) + res = ExecSshOnVmThroughGatewayVm(vmIp, partitionCommands) fmt.Println("Partitioning Results:", res) // Mount the new partition @@ -767,7 +767,7 @@ func FormatNVerifyPvcIsAccessible(diskUuid string, mountIndex int, vmIp string) fmt.Sprintf("grep -c ext4 %s/fstype", volFolder), "sync", } - results = execSshOnVmThroughGatewayVm(vmIp, mountCommands) + results = ExecSshOnVmThroughGatewayVm(vmIp, mountCommands) fmt.Println("Mounting Results:", results) // Verify the filesystem type @@ -778,7 +778,7 @@ func FormatNVerifyPvcIsAccessible(diskUuid string, mountIndex int, vmIp string) // verifyDataIntegrityOnVmDisk verifies data integrity with 100m random data on given FS path inside a vm func VerifyDataIntegrityOnVmDisk(vmIp, volFolder string) { - results := execSshOnVmThroughGatewayVm(vmIp, []string{"dd count=100 bs=1M if=/dev/urandom of=/tmp/file1", + results := ExecSshOnVmThroughGatewayVm(vmIp, []string{"dd count=100 bs=1M if=/dev/urandom of=/tmp/file1", "dd count=100 bs=1M if=/tmp/file1 of=" + volFolder + "/vmfile", "dd count=100 bs=1M if=" + volFolder + "/vmfile of=/tmp/file2", "md5sum /tmp/file1 /tmp/file2", }) @@ -786,8 +786,8 @@ func VerifyDataIntegrityOnVmDisk(vmIp, volFolder string) { gomega.Expect(strings.Fields(lines[0])[0]).To(gomega.Equal(strings.Fields(lines[1])[0])) } -// execSshOnVmThroughGatewayVm executes cmd(s) on VM via gateway(bastion) host and returns the result(s) -func execSshOnVmThroughGatewayVm(vmIp string, cmds []string) []fssh.Result { +// ExecSshOnVmThroughGatewayVm executes cmd(s) on VM via gateway(bastion) host and returns the result(s) +func ExecSshOnVmThroughGatewayVm(vmIp string, cmds []string) []fssh.Result { results := []fssh.Result{} gatewayClient, sshClient := GetSshClientForVmThroughGatewayVm(vmIp) @@ -928,7 +928,7 @@ func Wait4PvcAttachmentFailure( // mountFormattedVol2Vm mounts a preformatted volume inside the VM func MountFormattedVol2Vm(diskUuid string, mountIndex int, vmIp string) string { p := "/dev/disk/by-id/wwn-0x" + strings.ReplaceAll(strings.ToLower(diskUuid), "-", "") - results := execSshOnVmThroughGatewayVm(vmIp, []string{"ls -l /dev/disk/by-id/", "ls -l " + p}) + results := ExecSshOnVmThroughGatewayVm(vmIp, []string{"ls -l /dev/disk/by-id/", "ls -l " + p}) dev := "/dev/" + strings.TrimSpace(strings.Split(results[1].Stdout, "/")[6]) gomega.Expect(dev).ShouldNot(gomega.Equal("/dev/")) framework.Logf("Found %s dev for disk with uuid %s", dev, diskUuid) @@ -937,7 +937,7 @@ func MountFormattedVol2Vm(diskUuid string, mountIndex int, vmIp string) string { volMountPath := "/mnt/volume" + strconv.Itoa(mountIndex) volFolder := volMountPath + "/data" - results = execSshOnVmThroughGatewayVm(vmIp, []string{ + results = ExecSshOnVmThroughGatewayVm(vmIp, []string{ "sudo mkdir -p " + volMountPath, "sudo mount " + partitionDev + " " + volMountPath, "sudo chmod -R 777 " + volFolder,