diff --git a/manifests/supervisorcluster/1.29/cns-csi.yaml b/manifests/supervisorcluster/1.29/cns-csi.yaml index 5cba063c27..d19efe23ff 100644 --- a/manifests/supervisorcluster/1.29/cns-csi.yaml +++ b/manifests/supervisorcluster/1.29/cns-csi.yaml @@ -120,6 +120,9 @@ rules: - apiGroups: ["nsx.vmware.com"] resources: ["namespacenetworkinfos"] verbs: ["get", "list"] + - apiGroups: ["cns.vmware.com"] + resources: ["cnsvolumeattachments"] + verbs: ["create", "get", "list", "update", "delete", "watch", "patch"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 diff --git a/manifests/supervisorcluster/1.30/cns-csi.yaml b/manifests/supervisorcluster/1.30/cns-csi.yaml index eed62386c7..159c489677 100644 --- a/manifests/supervisorcluster/1.30/cns-csi.yaml +++ b/manifests/supervisorcluster/1.30/cns-csi.yaml @@ -123,6 +123,9 @@ rules: - apiGroups: ["nsx.vmware.com"] resources: ["namespacenetworkinfos"] verbs: ["get", "list"] + - apiGroups: ["cns.vmware.com"] + resources: ["cnsvolumeattachments"] + verbs: ["create", "get", "list", "update", "delete", "watch", "patch"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 diff --git a/manifests/supervisorcluster/1.31/cns-csi.yaml b/manifests/supervisorcluster/1.31/cns-csi.yaml index e2c19f1422..f05bbee8cf 100644 --- a/manifests/supervisorcluster/1.31/cns-csi.yaml +++ b/manifests/supervisorcluster/1.31/cns-csi.yaml @@ -123,6 +123,9 @@ rules: - apiGroups: ["nsx.vmware.com"] resources: ["namespacenetworkinfos"] verbs: ["get", "list"] + - apiGroups: ["cns.vmware.com"] + resources: ["cnsvolumeattachments"] + verbs: ["create", "get", "list", "update", "delete", "watch", "patch"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 diff --git a/manifests/supervisorcluster/1.32/cns-csi.yaml b/manifests/supervisorcluster/1.32/cns-csi.yaml index a6ba17302b..dcfa687d8b 100644 --- a/manifests/supervisorcluster/1.32/cns-csi.yaml +++ b/manifests/supervisorcluster/1.32/cns-csi.yaml @@ -123,6 +123,9 @@ rules: - apiGroups: ["nsx.vmware.com"] resources: ["namespacenetworkinfos"] verbs: ["get", "list"] + - apiGroups: ["cns.vmware.com"] + resources: ["cnsvolumeattachments"] + verbs: ["create", "get", "list", "update", "delete", "watch", "patch"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 diff --git a/pkg/internalapis/cnsoperator/cnsvolumeattachment/cnsvolumeattachment.go b/pkg/internalapis/cnsoperator/cnsvolumeattachment/cnsvolumeattachment.go new file mode 100644 index 0000000000..0f666f2898 --- /dev/null +++ b/pkg/internalapis/cnsoperator/cnsvolumeattachment/cnsvolumeattachment.go @@ -0,0 +1,298 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cnsvolumeattachment + +import ( + "context" + "fmt" + "sync" + + "k8s.io/apimachinery/pkg/api/errors" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/cache" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + k8s "sigs.k8s.io/vsphere-csi-driver/v3/pkg/kubernetes" + + "sigs.k8s.io/vsphere-csi-driver/v3/pkg/csi/service/logger" + "sigs.k8s.io/vsphere-csi-driver/v3/pkg/internalapis" + "sigs.k8s.io/vsphere-csi-driver/v3/pkg/internalapis/cnsoperator/cnsvolumeattachment/v1alpha1" + cnsoperatortypes "sigs.k8s.io/vsphere-csi-driver/v3/pkg/syncer/cnsoperator/types" +) + +var ( + cnsVolumeAttachmentInstanceLock sync.Mutex + cnsVolumeAttachmentInstance *cnsVolumeAttachment +) + +// CnsVolumeAttachment exposes an interface to support adding +// and removing information about attached VMs to a PVC. +type CnsVolumeAttachment interface { + // AddVmToAttachedList adds the input VM instance UUID to the list of + // attached VMs for the given volume. + AddVmToAttachedList(ctx context.Context, volumeName, VmInstanceUUID string) error + // RemoveVmFromAttachedList removes the input instance VM UUID from + // the list of attached VMs for the given volume. + RemoveVmFromAttachedList(ctx context.Context, volumeName, VmInstanceUUID string) (error, bool) +} + +// cnsVolumeAttachment maintains a client to the API +// server for operations on CnsVolumeAttachment instance. +// It also contains a per instance lock to handle +// concurrent operations. +type cnsVolumeAttachment struct { + client client.Client + // Per volume lock for concurrent access to CnsVolumeAttachment instances. + // Keys are strings representing PVC names. + // Values are individual sync.Mutex locks that need to be held + // to make updates to the CnsVolumeAttachment instance on the API server. + volumeLock *sync.Map +} + +// GetCnsVolumeAttachmentInstance returns a singleton of type CnsVolumeAttachment. +// Initializes the singleton if not already initialized. +func GetCnsVolumeAttachmentInstance(ctx context.Context) (CnsVolumeAttachment, error) { + log := logger.GetLogger(ctx) + + cnsVolumeAttachmentInstanceLock.Lock() + log.Infof("Acquired lock for cnsVolumeAttachmentInstanceLock") + defer func() { + cnsVolumeAttachmentInstanceLock.Unlock() + log.Infof("Released lock for cnsVolumeAttachmentInstanceLock") + }() + + if cnsVolumeAttachmentInstance == nil { + config, err := k8s.GetKubeConfig(ctx) + if err != nil { + log.Errorf("failed to get kubeconfig. Err: %v", err) + return nil, err + } + k8sclient, err := k8s.NewClientForGroup(ctx, config, internalapis.GroupName) + if err != nil { + log.Errorf("failed to create k8s client. Err: %v", err) + return nil, err + } + cnsVolumeAttachmentInstance = &cnsVolumeAttachment{ + client: k8sclient, + volumeLock: &sync.Map{}, + } + } + + return cnsVolumeAttachmentInstance, nil +} + +// Add adds the input VM InstanceUUID to the list of +// attached VMs for the given volume. +// Callers need to specify cnsVolumeAttachment as a combination of +// "/". This combination is used to uniquely +// identify CnsVolumeAttachment instances. +// The instance is created if it doesn't exist. +// Returns an error if the operation cannot be persisted on the API server. +func (f *cnsVolumeAttachment) AddVmToAttachedList(ctx context.Context, + volumeName, VmInstanceUUID string) error { + log := logger.GetLogger(ctx) + + log.Infof("Adding VM %s to cnsVolumeAttachment %s", + VmInstanceUUID, volumeName) + actual, _ := f.volumeLock.LoadOrStore(volumeName, &sync.Mutex{}) + instanceLock, ok := actual.(*sync.Mutex) + if !ok { + return fmt.Errorf("failed to cast lock for cnsVolumeAttachment instance: %s", volumeName) + } + instanceLock.Lock() + log.Infof("Acquired lock for cnsVolumeAttachment instance %s", volumeName) + defer func() { + instanceLock.Unlock() + log.Infof("Released lock for instance %s", volumeName) + }() + + instance := &v1alpha1.CnsVolumeAttachment{} + instanceNamespace, instanceName, err := cache.SplitMetaNamespaceKey(volumeName) + if err != nil { + log.Errorf("failed to split key %s with error: %+v", volumeName, err) + return err + } + instanceKey := types.NamespacedName{ + Namespace: instanceNamespace, + Name: instanceName, + } + err = f.client.Get(ctx, instanceKey, instance) + if err != nil { + if errors.IsNotFound(err) { + // Create the instance as it does not exist on the API server. + instance = &v1alpha1.CnsVolumeAttachment{ + ObjectMeta: v1.ObjectMeta{ + Name: instanceName, + Namespace: instanceNamespace, + // Add finalizer so that CnsVolumeAttachment instance doesn't get deleted abruptly + Finalizers: []string{cnsoperatortypes.CNSFinalizer}, + }, + Spec: v1alpha1.CnsVolumeAttachmentSpec{ + AttachedVms: []string{ + VmInstanceUUID, + }, + }, + } + log.Debugf("Creating cnsVolumeAttachment instance %s with spec: %+v", volumeName, instance) + err = f.client.Create(ctx, instance) + if err != nil { + log.Errorf("failed to create cnsVolumeAttachment instance %s with error: %+v", volumeName, err) + return err + } + return nil + } + log.Errorf("failed to get cnsVolumeAttachment instance %s with error: %+v", volumeName, err) + return err + } + + // Verify if input VmInstanceUUID exists in existing AttachedVMs list. + log.Debugf("Verifying if VM %s exists in current list of attached Vms. Current list: %+v", + VmInstanceUUID, instance.Spec.AttachedVms) + currentAttachedVmsList := instance.Spec.AttachedVms + for _, currentAttachedVM := range currentAttachedVmsList { + if currentAttachedVM == VmInstanceUUID { + log.Debugf("Found VM %s in list. Returning.", VmInstanceUUID) + return nil + } + } + newAttachVmsList := append(currentAttachedVmsList, VmInstanceUUID) + instance.Spec.AttachedVms = newAttachVmsList + log.Debugf("Updating cnsVolumeAttachment instance %s with spec: %+v", volumeName, instance) + err = f.client.Update(ctx, instance) + if err != nil { + log.Errorf("failed to update cnsVolumeAttachment instance %s/%s with error: %+v", volumeName, err) + } + return err +} + +// RemoveVmFromAttachedList removes the input VM UUID from +// the list of attached VMs for the given volume. +// Callers need to specify volumeName as a combination of +// "/". This combination is used to uniquely +// identify CnsVolumeAttachment instances. +// If the given VM was the last client for this file volume, the instance is +// deleted from the API server. +// Returns an error if the operation cannot be persisted on the API server. +func (f *cnsVolumeAttachment) RemoveVmFromAttachedList(ctx context.Context, + volumeName, VmInstanceUUID string) (error, bool) { + log := logger.GetLogger(ctx) + log.Infof("Removing VmInstanceUUID %s from cnsVolumeAttachment %s", + VmInstanceUUID, volumeName) + actual, _ := f.volumeLock.LoadOrStore(volumeName, &sync.Mutex{}) + instanceLock, ok := actual.(*sync.Mutex) + if !ok { + return fmt.Errorf("failed to cast lock for cnsVolumeAttachment instance: %s", volumeName), + false + } + instanceLock.Lock() + log.Infof("Acquired lock for cnsVolumeAttachment instance %s", volumeName) + defer func() { + instanceLock.Unlock() + log.Infof("Released lock for instance %s", volumeName) + }() + + instance := &v1alpha1.CnsVolumeAttachment{} + instanceNamespace, instanceName, err := cache.SplitMetaNamespaceKey(volumeName) + if err != nil { + log.Errorf("failed to split key %s with error: %+v", volumeName, err) + return err, false + } + instanceKey := types.NamespacedName{ + Namespace: instanceNamespace, + Name: instanceName, + } + err = f.client.Get(ctx, instanceKey, instance) + if err != nil { + if errors.IsNotFound(err) { + log.Infof("cnsVolumeAttachment instance %s does not exist on API server", volumeName) + return nil, true + } + log.Errorf("failed to get cnsVolumeAttachment instance %s with error: %+v", volumeName, err) + return err, false + } + + log.Infof("Verifying if VM UUID %s exists in list of already attached VMs. Current list: %+v", + volumeName, instance.Spec.AttachedVms) + for index, existingAttachedVM := range instance.Spec.AttachedVms { + if VmInstanceUUID != existingAttachedVM { + continue + } + log.Infof("Removing VmUUID %s from Attached VMs list", VmInstanceUUID) + instance.Spec.AttachedVms = append( + instance.Spec.AttachedVms[:index], + instance.Spec.AttachedVms[index+1:]...) + if len(instance.Spec.AttachedVms) == 0 { + log.Infof("Deleting cnsVolumeAttachment instance %s from API server", volumeName) + // Remove finalizer from CnsVolumeAttachment instance + err = removeFinalizer(ctx, f.client, instance) + if err != nil { + log.Errorf("failed to remove finalizer from cnsVolumeAttachment instance %s with error: %+v", + volumeName, err) + return err, false + } + err = f.client.Delete(ctx, instance) + if err != nil { + // In case of namespace deletion, we will have deletion timestamp added on the + // CnsVolumeAttachment instance. So, as soon as we delete finalizer, instance might + // get deleted immediately. In such cases we will get NotFound error here, return success + // if instance is already deleted. + if errors.IsNotFound(err) { + log.Infof("cnsVolumeAttachment instance %s seems to be already deleted.", volumeName) + f.volumeLock.Delete(volumeName) + return nil, true + } + log.Errorf("failed to delete cnsVolumeAttachment instance %s with error: %+v", volumeName, err) + return err, false + } + log.Infof("Successfully deleted cnsVolumeAttachment instance %s", volumeName) + f.volumeLock.Delete(volumeName) + return nil, true + } + log.Debugf("Updating cnsVolumeAttachment instance %s with spec: %+v", volumeName, instance) + err = f.client.Update(ctx, instance) + if err != nil { + log.Errorf("failed to update cnsVolumeAttachment instance %s with error: %+v", volumeName, err) + } + return err, false + } + log.Infof("Could not find VM %s in list. Returning.", VmInstanceUUID) + return nil, false +} + +// removeFinalizer will remove the CNS Finalizer = cns.vmware.com, +// from a given CnsVolumeAttachment instance. +func removeFinalizer(ctx context.Context, client client.Client, + instance *v1alpha1.CnsVolumeAttachment) error { + log := logger.GetLogger(ctx) + + if !controllerutil.ContainsFinalizer(instance, cnsoperatortypes.CNSFinalizer) { + // Finalizer not present on instance. Nothing to do. + return nil + } + + finalizersOnInstance := instance.Finalizers + for i, finalizer := range instance.Finalizers { + if finalizer == cnsoperatortypes.CNSFinalizer { + log.Infof("Removing %q finalizer from CnsNodeVmBatchAttachment instance with name: %q on namespace: %q", + cnsoperatortypes.CNSFinalizer, instance.Name, instance.Namespace) + finalizersOnInstance = append(instance.Finalizers[:i], instance.Finalizers[i+1:]...) + break + } + } + return k8s.PatchFinalizers(ctx, client, instance, finalizersOnInstance) +} diff --git a/pkg/internalapis/cnsoperator/cnsvolumeattachment/cnsvolumeattachment_test.go b/pkg/internalapis/cnsoperator/cnsvolumeattachment/cnsvolumeattachment_test.go new file mode 100644 index 0000000000..cbcd3b31d3 --- /dev/null +++ b/pkg/internalapis/cnsoperator/cnsvolumeattachment/cnsvolumeattachment_test.go @@ -0,0 +1,138 @@ +package cnsvolumeattachment + +import ( + "context" + "sync" + "testing" + + "github.com/stretchr/testify/assert" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/kubectl/pkg/scheme" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + cnsvolumeattachmentv1alpha1 "sigs.k8s.io/vsphere-csi-driver/v3/pkg/internalapis/cnsoperator/cnsvolumeattachment/v1alpha1" +) + +var ( + testCnsvolumeAttachmentName = "testvol" + testNamespace = "test-ns" +) + +func setupTestCnsNodeVmBatchAttachment() cnsvolumeattachmentv1alpha1.CnsVolumeAttachment { + + return cnsvolumeattachmentv1alpha1.CnsVolumeAttachment{ + ObjectMeta: metav1.ObjectMeta{ + Name: testCnsvolumeAttachmentName, + Namespace: testNamespace, + }, + Spec: cnsvolumeattachmentv1alpha1.CnsVolumeAttachmentSpec{ + AttachedVms: []string{ + "vm-1", + "vm-2", + }, + }, + } + +} + +func setTestEnvironment(testCnsVolumeAttachment cnsvolumeattachmentv1alpha1.CnsVolumeAttachment) client.WithWatch { + cnsVolumeAttachment := testCnsVolumeAttachment.DeepCopy() + objs := []runtime.Object{cnsVolumeAttachment} + + SchemeGroupVersion := schema.GroupVersion{ + Group: "cns.vmware.com", + Version: "v1alpha1", + } + + // Register operator types with the runtime scheme. + s := scheme.Scheme + s.AddKnownTypes(SchemeGroupVersion, cnsVolumeAttachment) + + fakeClient := fake.NewClientBuilder(). + WithScheme(s). + WithRuntimeObjects(objs...). + Build() + + return fakeClient + +} +func TestAddVmToAttachedList(t *testing.T) { + testCnsvolumeAttachment := setupTestCnsNodeVmBatchAttachment() + fakeClient := setTestEnvironment(testCnsvolumeAttachment) + + t.Run("TestAddVmToAttachedList", func(t *testing.T) { + cnsAttachmentInstance := &cnsVolumeAttachment{ + client: fakeClient, + volumeLock: &sync.Map{}, + } + + volumeName := testNamespace + "/" + testCnsvolumeAttachmentName + addVmToAttachedList := "vm-3" + err := cnsAttachmentInstance.AddVmToAttachedList(context.Background(), volumeName, addVmToAttachedList) + assert.NoError(t, err) + + namespacedName := types.NamespacedName{ + Name: testCnsvolumeAttachmentName, + Namespace: testNamespace, + } + + updatedCnsVolumeAttachment := &cnsvolumeattachmentv1alpha1.CnsVolumeAttachment{} + if err := fakeClient.Get(context.TODO(), namespacedName, updatedCnsVolumeAttachment); err != nil { + t.Fatalf("get updatedCnsVolumeAttachment: %v", err) + } + + exepectedAttachedVmsList := []string{"vm-1", "vm-2", "vm-3"} + + assert.Equal(t, exepectedAttachedVmsList, updatedCnsVolumeAttachment.Spec.AttachedVms) + }) + +} + +func TestRemoveVmFromAttachedList(t *testing.T) { + testCnsvolumeAttachment := setupTestCnsNodeVmBatchAttachment() + fakeClient := setTestEnvironment(testCnsvolumeAttachment) + + t.Run("TestRemoveVmFromAttachedList", func(t *testing.T) { + cnsAttachmentInstance := &cnsVolumeAttachment{ + client: fakeClient, + volumeLock: &sync.Map{}, + } + + // Remove VM-2 + volumeName := testNamespace + "/" + testCnsvolumeAttachmentName + removeVmFromAttachedList := "vm-2" + err, isLastAttachedVm := cnsAttachmentInstance.RemoveVmFromAttachedList(context.Background(), + volumeName, removeVmFromAttachedList) + assert.NoError(t, err) + assert.Equal(t, false, isLastAttachedVm) + + namespacedName := types.NamespacedName{ + Name: testCnsvolumeAttachmentName, + Namespace: testNamespace, + } + + updatedCnsVolumeAttachment := &cnsvolumeattachmentv1alpha1.CnsVolumeAttachment{} + if err := fakeClient.Get(context.TODO(), namespacedName, updatedCnsVolumeAttachment); err != nil { + t.Fatalf("get updatedCnsVolumeAttachment: %v", err) + } + + exepectedAttachedVmsList := []string{"vm-1"} + assert.Equal(t, exepectedAttachedVmsList, updatedCnsVolumeAttachment.Spec.AttachedVms) + + // Remove VM-1 also + removeVmFromAttachedList = "vm-1" + err, isLastAttachedVm = cnsAttachmentInstance.RemoveVmFromAttachedList(context.Background(), + volumeName, removeVmFromAttachedList) + assert.NoError(t, err) + assert.Equal(t, true, isLastAttachedVm) + + updatedCnsVolumeAttachment = &cnsvolumeattachmentv1alpha1.CnsVolumeAttachment{} + err = fakeClient.Get(context.TODO(), namespacedName, updatedCnsVolumeAttachment) + assert.Error(t, err) + + }) +} diff --git a/pkg/internalapis/cnsoperator/cnsvolumeattachment/mock.go b/pkg/internalapis/cnsoperator/cnsvolumeattachment/mock.go new file mode 100644 index 0000000000..10f4124f33 --- /dev/null +++ b/pkg/internalapis/cnsoperator/cnsvolumeattachment/mock.go @@ -0,0 +1,17 @@ +package cnsvolumeattachment + +import "context" + +type MockCnsVolumeAttachment struct{} + +func (m *MockCnsVolumeAttachment) AddVmToAttachedList(ctx context.Context, volumeName, + VmInstanceUUID string) error { + // Mock behavior + return nil +} + +func (m *MockCnsVolumeAttachment) RemoveVmFromAttachedList(ctx context.Context, volumeName, + VmInstanceUUID string) (error, bool) { + // Mock behavior + return nil, true +} diff --git a/pkg/internalapis/cnsoperator/cnsvolumeattachment/v1alpha1/cnsvolumeattachment_types.go b/pkg/internalapis/cnsoperator/cnsvolumeattachment/v1alpha1/cnsvolumeattachment_types.go new file mode 100644 index 0000000000..f506c8a493 --- /dev/null +++ b/pkg/internalapis/cnsoperator/cnsvolumeattachment/v1alpha1/cnsvolumeattachment_types.go @@ -0,0 +1,48 @@ +/* +Copyright 2025 The Kubernetes authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// CnsVolumeAttachmentSpec contains the list of VM UUIDs attached +// to the given volume +type CnsVolumeAttachmentSpec struct { + AttachedVms []string `json:"attachedVms,omitempty"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status + +// CnsVolumeAttachment is the Schema for the cnsvolumeattachment CRD. This CRD is used by +// CNS-CSI for internal bookkeeping purposes only and is not an API. +type CnsVolumeAttachment struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec CnsVolumeAttachmentSpec `json:"spec,omitempty"` +} + +// +kubebuilder:object:root=true + +// CnsVolumeAttachmentList contains a list of CnsVolumeAttachment +type CnsVolumeAttachmentList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []CnsVolumeAttachment `json:"items"` +} diff --git a/pkg/internalapis/cnsoperator/cnsvolumeattachment/v1alpha1/doc.go b/pkg/internalapis/cnsoperator/cnsvolumeattachment/v1alpha1/doc.go new file mode 100644 index 0000000000..cbeb1f96c7 --- /dev/null +++ b/pkg/internalapis/cnsoperator/cnsvolumeattachment/v1alpha1/doc.go @@ -0,0 +1,5 @@ +// +k8s:deepcopy-gen=package +// +k8s:defaulter-gen=TypeMeta +// +groupName=cns.vmware.com + +package v1alpha1 diff --git a/pkg/internalapis/cnsoperator/cnsvolumeattachment/v1alpha1/zz_generated.deepcopy.go b/pkg/internalapis/cnsoperator/cnsvolumeattachment/v1alpha1/zz_generated.deepcopy.go new file mode 100644 index 0000000000..5fc90e8633 --- /dev/null +++ b/pkg/internalapis/cnsoperator/cnsvolumeattachment/v1alpha1/zz_generated.deepcopy.go @@ -0,0 +1,105 @@ +// build : ignore_autogenerated + +/* +Copyright 2025 The Kubernetes authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by deepcopy-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CnsVolumeAttachment) DeepCopyInto(out *CnsVolumeAttachment) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CnsVolumeAttachment. +func (in *CnsVolumeAttachment) DeepCopy() *CnsVolumeAttachment { + if in == nil { + return nil + } + out := new(CnsVolumeAttachment) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CnsVolumeAttachment) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CnsVolumeAttachmentList) DeepCopyInto(out *CnsVolumeAttachmentList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]CnsVolumeAttachment, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CnsVolumeAttachmentList. +func (in *CnsVolumeAttachmentList) DeepCopy() *CnsVolumeAttachmentList { + if in == nil { + return nil + } + out := new(CnsVolumeAttachmentList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CnsVolumeAttachmentList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CnsVolumeAttachmentSpec) DeepCopyInto(out *CnsVolumeAttachmentSpec) { + *out = *in + if in.AttachedVms != nil { + in, out := &in.AttachedVms, &out.AttachedVms + *out = make([]string, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CnsVolumeAttachmentSpec. +func (in *CnsVolumeAttachmentSpec) DeepCopy() *CnsVolumeAttachmentSpec { + if in == nil { + return nil + } + out := new(CnsVolumeAttachmentSpec) + in.DeepCopyInto(out) + return out +} diff --git a/pkg/internalapis/cnsoperator/config/cnsvolumeattachment_crd.yaml b/pkg/internalapis/cnsoperator/config/cnsvolumeattachment_crd.yaml new file mode 100644 index 0000000000..a7f06ff2c1 --- /dev/null +++ b/pkg/internalapis/cnsoperator/config/cnsvolumeattachment_crd.yaml @@ -0,0 +1,55 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + name: cnsvolumeattachments.cns.vmware.com +spec: + group: cns.vmware.com + names: + kind: CnsVolumeAttachment + listKind: CnsVolumeAttachmentList + plural: cnsvolumeattachments + singular: cnsvolumeattachment + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + CnsVolumeAttachment is the Schema for the cnsvolumeattachment CRD. This CRD is used by + CNS-CSI for internal bookkeeping purposes only and is not an API. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + CnsVolumeAttachmentSpec contains the list of VM UUIDs attached + to the given volume + properties: + attachedVms: + items: + type: string + type: array + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/pkg/internalapis/cnsoperator/config/config.go b/pkg/internalapis/cnsoperator/config/config.go index 30bf8042da..1645cdfbd6 100644 --- a/pkg/internalapis/cnsoperator/config/config.go +++ b/pkg/internalapis/cnsoperator/config/config.go @@ -7,6 +7,11 @@ var EmbedCnsFileVolumeClientFile embed.FS const EmbedCnsFileVolumeClientFileName = "cnsfilevolumeclient_crd.yaml" +//go:embed cnsvolumeattachment_crd.yaml +var EmbedCnsVolumeAttachmentFile embed.FS + +const EmbedCnsVolumeAttachmentFileName = "cnsvolumeattachment_crd.yaml" + //go:embed triggercsifullsync_crd.yaml var EmbedTriggerCsiFullSync embed.FS diff --git a/pkg/internalapis/register.go b/pkg/internalapis/register.go index 2ca988506a..040f8b8372 100644 --- a/pkg/internalapis/register.go +++ b/pkg/internalapis/register.go @@ -27,6 +27,8 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" cnsfilevolclientv1alpha1 "sigs.k8s.io/vsphere-csi-driver/v3/pkg/internalapis/cnsoperator/cnsfilevolumeclient/v1alpha1" + cnsvolumeattachmentv1alpha1 "sigs.k8s.io/vsphere-csi-driver/v3/pkg/internalapis/cnsoperator/cnsvolumeattachment/v1alpha1" + triggercsifullsyncv1alpha1 "sigs.k8s.io/vsphere-csi-driver/v3/pkg/internalapis/cnsoperator/triggercsifullsync/v1alpha1" cnscsisvfeaturestatesv1alpha1 "sigs.k8s.io/vsphere-csi-driver/v3/pkg/internalapis/featurestates/v1alpha1" ) @@ -74,6 +76,12 @@ func addKnownTypes(scheme *runtime.Scheme) error { &cnsfilevolclientv1alpha1.CnsFileVolumeClientList{}, ) + scheme.AddKnownTypes( + SchemeGroupVersion, + &cnsvolumeattachmentv1alpha1.CnsVolumeAttachment{}, + &cnsvolumeattachmentv1alpha1.CnsVolumeAttachmentList{}, + ) + scheme.AddKnownTypes( SchemeGroupVersion, &triggercsifullsyncv1alpha1.TriggerCsiFullSync{}, diff --git a/pkg/syncer/cnsoperator/controller/cnsnodevmbatchattachment/cnsnodevmbatchattachment_controller.go b/pkg/syncer/cnsoperator/controller/cnsnodevmbatchattachment/cnsnodevmbatchattachment_controller.go index 4d41cf2691..0d1e9cd162 100644 --- a/pkg/syncer/cnsoperator/controller/cnsnodevmbatchattachment/cnsnodevmbatchattachment_controller.go +++ b/pkg/syncer/cnsoperator/controller/cnsnodevmbatchattachment/cnsnodevmbatchattachment_controller.go @@ -30,6 +30,7 @@ import ( cnsvsphere "sigs.k8s.io/vsphere-csi-driver/v3/pkg/common/cns-lib/vsphere" "sigs.k8s.io/vsphere-csi-driver/v3/pkg/csi/service/common/commonco" + "sigs.k8s.io/vsphere-csi-driver/v3/pkg/internalapis/cnsoperator/cnsvolumeattachment" cnsoperatortypes "sigs.k8s.io/vsphere-csi-driver/v3/pkg/syncer/cnsoperator/types" vmoperatorv1alpha4 "github.com/vmware-tanzu/vm-operator/api/v1alpha4" @@ -64,8 +65,9 @@ import ( // reconcile operation succeeded. // If the reconcile fails, backoff is incremented exponentially. var ( - backOffDuration map[types.NamespacedName]time.Duration - backOffDurationMapMutex = sync.Mutex{} + backOffDuration map[types.NamespacedName]time.Duration + backOffDurationMapMutex = sync.Mutex{} + GetCnsVolumeAttachmentInstanceFunc = cnsvolumeattachment.GetCnsVolumeAttachmentInstance ) const ( @@ -300,7 +302,22 @@ func (r *Reconciler) Reconcile(ctx context.Context, // This means all volumes can be considered detached. So remove finalizer from CR instance. if instance.DeletionTimestamp != nil && vm == nil { log.Infof("Instance %s is being deleted and VM object is also deleted from VC", request.NamespacedName.String()) - // TODO: remove PVC finalizer + cnsVolumeAttachmentInstance, err := GetCnsVolumeAttachmentInstanceFunc(ctx) + if err != nil { + log.Errorf("failed to get cnsvolumeattachment singleton instance. Err: %s", err) + return r.completeReconciliationWithError(batchAttachCtx, instance, request.NamespacedName, timeout, err) + } + + // For every PV mentioned in instance.Spec, remove finalizer from it. + for _, volume := range instance.Spec.Volumes { + err := removePvcFinalizerFunc(ctx, r.client, volume.PersistentVolumeClaim.ClaimName, instance.Namespace, + instance.Spec.NodeUUID, cnsVolumeAttachmentInstance) + if err != nil { + log.Errorf("failed to remove finalizer from PVC %s. Err: %s", volume.PersistentVolumeClaim.ClaimName, + err) + return r.completeReconciliationWithError(batchAttachCtx, instance, request.NamespacedName, timeout, err) + } + } patchErr := removeFinalizerFromCRDInstance(batchAttachCtx, instance, r.client) if patchErr != nil { @@ -373,7 +390,12 @@ func (r *Reconciler) reconcileInstanceWithDeletionTimestamp(ctx context.Context, vm *cnsvsphere.VirtualMachine) error { log := logger.GetLogger(ctx) - err := r.processDetach(ctx, vm, instance, volumesToDetach) + cnsVolumeAttachmentInstance, err := GetCnsVolumeAttachmentInstanceFunc(ctx) + if err != nil { + return logger.LogNewErrorf(log, "Failed to get CNSFileVolumeClient instance. Error: %+v", err) + } + + err = r.processDetach(ctx, vm, instance, volumesToDetach, cnsVolumeAttachmentInstance) if err != nil { log.Errorf("failed to detach all volumes. Err: %s", err) return err @@ -391,8 +413,13 @@ func (r *Reconciler) reconcileInstanceWithoutDeletionTimestamp(ctx context.Conte vm *cnsvsphere.VirtualMachine) error { log := logger.GetLogger(ctx) + cnsVolumeAttachmentInstance, err := GetCnsVolumeAttachmentInstanceFunc(ctx) + if err != nil { + return logger.LogNewErrorf(log, "Failed to get CNSFileVolumeClient instance. Error: %+v", err) + } + // Call batch attach for volumes. - err := r.processBatchAttach(ctx, vm, instance) + err = r.processBatchAttach(ctx, vm, instance, cnsVolumeAttachmentInstance) if err != nil { log.Errorf("failed to attach all volumes. Err: %+v", err) return err @@ -400,7 +427,7 @@ func (r *Reconciler) reconcileInstanceWithoutDeletionTimestamp(ctx context.Conte // Call detach if there are some volumes which need to be detached. if len(volumesToDetach) != 0 { - err := r.processDetach(ctx, vm, instance, volumesToDetach) + err := r.processDetach(ctx, vm, instance, volumesToDetach, cnsVolumeAttachmentInstance) if err != nil { log.Errorf("failed to detach all volumes. Err: +v", err) return err @@ -413,11 +440,13 @@ func (r *Reconciler) reconcileInstanceWithoutDeletionTimestamp(ctx context.Conte // processDetach detaches each of the volumes in volumesToDetach by calling CNS DetachVolume API. func (r *Reconciler) processDetach(ctx context.Context, vm *cnsvsphere.VirtualMachine, - instance *v1alpha1.CnsNodeVmBatchAttachment, volumesToDetach map[string]string) error { + instance *v1alpha1.CnsNodeVmBatchAttachment, + volumesToDetach map[string]string, + cnsVolumeAttachmentInstance cnsvolumeattachment.CnsVolumeAttachment) error { log := logger.GetLogger(ctx) log.Debugf("Calling detach volume for PVC %+v", volumesToDetach) - volumesThatFailedToDetach := r.detachVolumes(ctx, vm, volumesToDetach, instance) + volumesThatFailedToDetach := r.detachVolumes(ctx, vm, volumesToDetach, instance, cnsVolumeAttachmentInstance) var overallErr error if len(volumesThatFailedToDetach) != 0 { @@ -434,7 +463,8 @@ func (r *Reconciler) processDetach(ctx context.Context, // detachVolumes calls Cns DetachVolume for every PVC in volumesToDetach. func (r *Reconciler) detachVolumes(ctx context.Context, vm *cnsvsphere.VirtualMachine, volumesToDetach map[string]string, - instance *v1alpha1.CnsNodeVmBatchAttachment) []string { + instance *v1alpha1.CnsNodeVmBatchAttachment, + cnsVolumeAttachmentInstance cnsvolumeattachment.CnsVolumeAttachment) []string { log := logger.GetLogger(ctx) volumesThatFailedToDetach := make([]string, 0) @@ -449,11 +479,19 @@ func (r *Reconciler) detachVolumes(ctx context.Context, // If VM was not found, can assume that the detach is successful. if cnsvsphere.IsManagedObjectNotFound(detachErr, vm.VirtualMachine.Reference()) { log.Infof("Found a managed object not found fault for vm: %+v", vm) - // TODO: remove PVC finalizer - // Remove entry of this volume from the instance's status. - deleteVolumeFromStatus(pvc, instance) - log.Infof("Successfully detached volume %s from VM %s", pvc, instance.Spec.NodeUUID) + // Remove finalizer from the PVC as the detach was successful. + err := removePvcFinalizerFunc(ctx, r.client, pvc, instance.Namespace, instance.Spec.NodeUUID, + cnsVolumeAttachmentInstance) + if err != nil { + log.Errorf("failed to rempve finalizer from PVC %s. Err: %s", pvc, err) + updateInstanceWithErrorForPvc(instance, pvc, err.Error()) + volumesThatFailedToDetach = append(volumesThatFailedToDetach, pvc) + } else { + // Remove entry of this volume from the instance's status. + deleteVolumeFromStatus(pvc, instance) + log.Infof("Successfully detached volume %s from VM %s", pvc, instance.Spec.NodeUUID) + } } else { log.Errorf("failed to detach volume %s from VM %s. Fault: %s Err: %s", pvc, instance.Spec.NodeUUID, faulttype, detachErr) @@ -462,10 +500,18 @@ func (r *Reconciler) detachVolumes(ctx context.Context, volumesThatFailedToDetach = append(volumesThatFailedToDetach, pvc) } } else { - // TODO: remove PVC finalizer - // Remove entry of this volume from the instance's status. - deleteVolumeFromStatus(pvc, instance) - log.Infof("Successfully detached volume %s from VM %s", pvc, instance.Spec.NodeUUID) + // Remove finalizer from the PVC as detach was successful. + err := removePvcFinalizerFunc(ctx, r.client, pvc, instance.Namespace, instance.Spec.NodeUUID, + cnsVolumeAttachmentInstance) + if err != nil { + log.Errorf("failed to remove finalizer from PVC %s. Err: %s", pvc, err) + updateInstanceWithErrorForPvc(instance, pvc, err.Error()) + volumesThatFailedToDetach = append(volumesThatFailedToDetach, pvc) + } else { + // Remove entry of this volume from the instance's status. + deleteVolumeFromStatus(pvc, instance) + log.Infof("Successfully detached volume %s from VM %s", pvc, instance.Spec.NodeUUID) + } } log.Infof("Detach call ended for PVC %s in namespace %s for instance %s", pvc, instance.Namespace, instance.Name) @@ -478,7 +524,8 @@ func (r *Reconciler) detachVolumes(ctx context.Context, // processBatchAttach first constructs the batch attach volume request for all volumes in instance spec // and then calls CNS batch attach for them. func (r *Reconciler) processBatchAttach(ctx context.Context, vm *cnsvsphere.VirtualMachine, - instance *v1alpha1.CnsNodeVmBatchAttachment) error { + instance *v1alpha1.CnsNodeVmBatchAttachment, + cnsVolumeAttachmentInstance cnsvolumeattachment.CnsVolumeAttachment) error { log := logger.GetLogger(ctx) // Construct batch attach request @@ -509,6 +556,19 @@ func (r *Reconciler) processBatchAttach(ctx context.Context, vm *cnsvsphere.Virt return fmt.Errorf("failed to get volumeName for pvc %s", pvcName) } + + // If attach was successful, add finalizer to the PVC. + if result.Error == nil { + // Add finalizer on PVC as attach was successful. + err = addPvcFinalizerFunc(ctx, r.client, pvcName, instance.Namespace, instance.Spec.NodeUUID, + cnsVolumeAttachmentInstance) + if err != nil { + log.Errorf("failed to add finalizer %s on PVC %s", cnsoperatortypes.CNSPvcFinalizer, pvcName) + result.Error = err + attachErr = err + } + } + // Update instance with attach result updateInstanceWithAttachVolumeResult(instance, volumeName, pvcName, result) } diff --git a/pkg/syncer/cnsoperator/controller/cnsnodevmbatchattachment/cnsnodevmbatchattachment_helper.go b/pkg/syncer/cnsoperator/controller/cnsnodevmbatchattachment/cnsnodevmbatchattachment_helper.go index 1d11dccfe7..5ae7ee94c5 100644 --- a/pkg/syncer/cnsoperator/controller/cnsnodevmbatchattachment/cnsnodevmbatchattachment_helper.go +++ b/pkg/syncer/cnsoperator/controller/cnsnodevmbatchattachment/cnsnodevmbatchattachment_helper.go @@ -25,19 +25,23 @@ import ( vimtypes "github.com/vmware/govmomi/vim25/types" v1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" v1alpha1 "sigs.k8s.io/vsphere-csi-driver/v3/pkg/apis/cnsoperator/cnsnodevmbatchattachment/v1alpha1" volumes "sigs.k8s.io/vsphere-csi-driver/v3/pkg/common/cns-lib/volume" cnsvsphere "sigs.k8s.io/vsphere-csi-driver/v3/pkg/common/cns-lib/vsphere" "sigs.k8s.io/vsphere-csi-driver/v3/pkg/common/config" "sigs.k8s.io/vsphere-csi-driver/v3/pkg/csi/service/common/commonco" "sigs.k8s.io/vsphere-csi-driver/v3/pkg/csi/service/logger" + "sigs.k8s.io/vsphere-csi-driver/v3/pkg/internalapis/cnsoperator/cnsvolumeattachment" k8s "sigs.k8s.io/vsphere-csi-driver/v3/pkg/kubernetes" cnsoperatortypes "sigs.k8s.io/vsphere-csi-driver/v3/pkg/syncer/cnsoperator/types" cnsoperatorutil "sigs.k8s.io/vsphere-csi-driver/v3/pkg/syncer/cnsoperator/util" ) var ( - GetVMFromVcenter = cnsoperatorutil.GetVMFromVcenter + GetVMFromVcenter = cnsoperatorutil.GetVMFromVcenter + removePvcFinalizerFunc = removePvcFinalizer + addPvcFinalizerFunc = addPvcFinalizer ) // removeFinalizerFromCRDInstance will remove the CNS Finalizer, cns.vmware.com, @@ -278,9 +282,7 @@ func getPvcsInSpec(instance *v1alpha1.CnsNodeVmBatchAttachment) (map[string]stri } pvcsInSpec[volume.PersistentVolumeClaim.ClaimName] = volumeId } - return pvcsInSpec, nil - } // listAttachedFcdsForVM returns list of FCDs (present in the K8s cluster) @@ -505,3 +507,85 @@ func getVolumesToDetachFromVM(ctx context.Context, client client.Client, return volumesToDetach, nil } + +// addPvcFinalizer adds the given VM to the PVC's attached lists and then adds finalzier on the PVC. +func addPvcFinalizer(ctx context.Context, client client.Client, + pvcName string, namespace string, vmInstanceUUID string, + cnsVolumeAttachmentInstance cnsvolumeattachment.CnsVolumeAttachment) error { + log := logger.GetLogger(ctx) + + // Get PVC object from informer cache + pvc, err := commonco.ContainerOrchestratorUtility.GetPvcObjectByName(ctx, pvcName, namespace) + if err != nil { + log.Errorf("failed to get PVC object for PVC %s. Err: %s", pvcName, err) + return err + } + + // Add VM to the list of attached VMs for the given PVC. + err = cnsVolumeAttachmentInstance.AddVmToAttachedList(ctx, namespace+"/"+pvcName, vmInstanceUUID) + if err != nil { + log.Errorf("failed to add VM %s to list of attached VMs for PVC %s in namespace %s", vmInstanceUUID, + pvcName, namespace) + return err + } + + // If finalizer already exists, there is nothing to be done. + if controllerutil.ContainsFinalizer(pvc, cnsoperatortypes.CNSPvcFinalizer) { + // Finalizer already present on PVC + return nil + } + + return k8s.PatchFinalizers(ctx, client, pvc, + append(pvc.Finalizers, cnsoperatortypes.CNSPvcFinalizer)) +} + +// removePvcFinalizer removed the given VM from the PVC's list of attached VMs +// and then remmoves finalizer from the PVC if it was the last attached VM for the PVC. +func removePvcFinalizer(ctx context.Context, client client.Client, + pvcName string, namespace string, vmInstanceUUID string, + cnsVolumeAttachmentInstance cnsvolumeattachment.CnsVolumeAttachment) error { + log := logger.GetLogger(ctx) + + // Get PVC object from informer cache + pvc, err := commonco.ContainerOrchestratorUtility.GetPvcObjectByName(ctx, pvcName, namespace) + if err != nil { + log.Errorf("failed to get PVC object for PVC %s. Err: %s", pvcName, err) + return err + } + + volumeName := namespace + "/" + pvcName + + // Remove the VM from list of attached VMs for this PVC. + // If it was the last attached VM, isLastAttachedVm will be true. + err, isLastAttachedVm := cnsVolumeAttachmentInstance.RemoveVmFromAttachedList(ctx, + volumeName, vmInstanceUUID) + if err != nil { + log.Errorf("failed to remove VM %s from attached list for PVC %s", vmInstanceUUID, pvcName) + return err + } + if !isLastAttachedVm { + // If it is not the last attached VM, do not remove finalizer from the PVC. + return nil + } + + log.Infof("VM %s was the last attached VM for the PVC %s. Finalzier %s can be safely removed fromt the PVC", + vmInstanceUUID, pvcName, namespace) + + if !controllerutil.ContainsFinalizer(pvc, cnsoperatortypes.CNSPvcFinalizer) { + // Finalizer not present on PVC, nothing to be done here. + return nil + } + + // Remove finalizer from the PVC if it was the last attached VM. + finalizersOnPvc := pvc.Finalizers + for i, finalizer := range pvc.Finalizers { + if finalizer == cnsoperatortypes.CNSFinalizer { + log.Infof("Removing %s finalizer from PVC: %s on namespace: %s", + cnsoperatortypes.CNSFinalizer, pvcName, namespace) + finalizersOnPvc = append(pvc.Finalizers[:i], pvc.Finalizers[i+1:]...) + break + } + } + + return k8s.PatchFinalizers(ctx, client, pvc, finalizersOnPvc) +} diff --git a/pkg/syncer/cnsoperator/controller/cnsnodevmbatchattachment/cnsnodevmbatchattachment_test.go b/pkg/syncer/cnsoperator/controller/cnsnodevmbatchattachment/cnsnodevmbatchattachment_test.go index e186ab4784..5849e54901 100644 --- a/pkg/syncer/cnsoperator/controller/cnsnodevmbatchattachment/cnsnodevmbatchattachment_test.go +++ b/pkg/syncer/cnsoperator/controller/cnsnodevmbatchattachment/cnsnodevmbatchattachment_test.go @@ -31,6 +31,7 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -41,6 +42,7 @@ import ( "sigs.k8s.io/vsphere-csi-driver/v3/pkg/csi/service/common/commonco" v1alpha1 "sigs.k8s.io/vsphere-csi-driver/v3/pkg/apis/cnsoperator/cnsnodevmbatchattachment/v1alpha1" + "sigs.k8s.io/vsphere-csi-driver/v3/pkg/internalapis/cnsoperator/cnsvolumeattachment" ) var ( @@ -203,6 +205,9 @@ func TestCnsNodeVmBatchAttachmentWhenVmOnVcenterReturnsNotFoundError(t *testing. GetVMFromVcenter = MockGetVMFromVcenter commonco.ContainerOrchestratorUtility = &unittestcommon.FakeK8SOrchestrator{} + GetCnsVolumeAttachmentInstanceFunc = MockGetCnsVolumeAttachmentInstance + removePvcFinalizerFunc = mockRemovePvcFinalizer + addPvcFinalizerFunc = mockAddPvcFinalizer res, err := r.Reconcile(context.TODO(), req) if err != nil { @@ -274,6 +279,9 @@ func TestReconcileWithDeletionTimestamp(t *testing.T) { r := setTestEnvironment(&testCnsNodeVmBatchAttachment, false) mockVolumeManager := &unittestcommon.MockVolumeManager{} r.volumeManager = mockVolumeManager + GetCnsVolumeAttachmentInstanceFunc = MockGetCnsVolumeAttachmentInstance + removePvcFinalizerFunc = mockRemovePvcFinalizer + addPvcFinalizerFunc = mockAddPvcFinalizer volumesToDetach := map[string]string{ "pvc-1": "123-456", @@ -292,6 +300,9 @@ func TestReconcileWithDeletionTimestampWhenDetachFails(t *testing.T) { r := setTestEnvironment(&testCnsNodeVmBatchAttachment, false) mockVolumeManager := &unittestcommon.MockVolumeManager{} r.volumeManager = mockVolumeManager + GetCnsVolumeAttachmentInstanceFunc = MockGetCnsVolumeAttachmentInstance + removePvcFinalizerFunc = mockRemovePvcFinalizer + addPvcFinalizerFunc = mockAddPvcFinalizer volumesToDetach := map[string]string{ "pvc-1": "fail-detach", @@ -322,6 +333,9 @@ func TestReconcileWithoutDeletionTimestamp(t *testing.T) { mockVolumeManager := &unittestcommon.MockVolumeManager{} r.volumeManager = mockVolumeManager commonco.ContainerOrchestratorUtility = &unittestcommon.FakeK8SOrchestrator{} + GetCnsVolumeAttachmentInstanceFunc = MockGetCnsVolumeAttachmentInstance + removePvcFinalizerFunc = mockRemovePvcFinalizer + addPvcFinalizerFunc = mockAddPvcFinalizer volumesToDetach := map[string]string{ "pvc-1": "123-456", @@ -342,6 +356,9 @@ func TestReconcileWithoutDeletionTimestampWhenAttachFails(t *testing.T) { mockVolumeManager := &unittestcommon.MockVolumeManager{} r.volumeManager = mockVolumeManager commonco.ContainerOrchestratorUtility = &unittestcommon.FakeK8SOrchestrator{} + GetCnsVolumeAttachmentInstanceFunc = MockGetCnsVolumeAttachmentInstance + removePvcFinalizerFunc = mockRemovePvcFinalizer + addPvcFinalizerFunc = mockAddPvcFinalizer volumesToDetach := map[string]string{ "pvc-1": "123-456", @@ -434,3 +451,19 @@ func MockGetVMFromVcenter(ctx context.Context, nodeUUID string, } return &cnsvsphere.VirtualMachine{}, nil } + +func MockGetCnsVolumeAttachmentInstance(ctx context.Context) (cnsvolumeattachment.CnsVolumeAttachment, error) { + return &cnsvolumeattachment.MockCnsVolumeAttachment{}, nil +} + +func mockRemovePvcFinalizer(ctx context.Context, client client.Client, + pvcName string, namespace string, vmInstanceUUID string, + cnsVolumeAttachmentInstance cnsvolumeattachment.CnsVolumeAttachment) error { + return nil +} + +func mockAddPvcFinalizer(ctx context.Context, client client.Client, + pvcName string, namespace string, vmUUID string, + cnsVolumeAttachmentInstance cnsvolumeattachment.CnsVolumeAttachment) error { + return nil +} diff --git a/pkg/syncer/cnsoperator/manager/init.go b/pkg/syncer/cnsoperator/manager/init.go index e82281058e..e6fe0508a1 100644 --- a/pkg/syncer/cnsoperator/manager/init.go +++ b/pkg/syncer/cnsoperator/manager/init.go @@ -255,6 +255,18 @@ func InitCnsOperator(ctx context.Context, clusterFlavor cnstypes.CnsClusterFlavo } } } + + if commonco.ContainerOrchestratorUtility.IsFSSEnabled(ctx, common.SharedDiskFss) { + // Create CnsVolumeAttachment CRD from manifest if shared disk feature + // is enabled. + err = k8s.CreateCustomResourceDefinitionFromManifest(ctx, + internalapiscnsoperatorconfig.EmbedCnsVolumeAttachmentFile, + internalapiscnsoperatorconfig.EmbedCnsVolumeAttachmentFileName) + if err != nil { + log.Errorf("Failed to create %q CRD. Err: %+v", internalapis.CnsFileVolumeClientPlural, err) + return err + } + } } else if clusterFlavor == cnstypes.CnsClusterFlavorVanilla { // Create CSINodeTopology CRD. err = k8s.CreateCustomResourceDefinitionFromManifest(ctx, csinodetopologyconfig.EmbedCSINodeTopologyFile,