Skip to content

Commit 8c3c35d

Browse files
skogtaskogta
committed
Add CnsSharedDiskVolumeClient CRD
1 parent 912cd98 commit 8c3c35d

File tree

8 files changed

+564
-0
lines changed

8 files changed

+564
-0
lines changed

‎pkg/internalapis/cnsoperator/cnsvolumeattachment/cnsvolumeattachment.go

Lines changed: 325 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,325 @@
1+
/*
2+
Copyright 2025 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package cnsvolumeattachment
18+
19+
import (
20+
"context"
21+
"fmt"
22+
"sync"
23+
24+
"k8s.io/apimachinery/pkg/api/errors"
25+
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
26+
"k8s.io/apimachinery/pkg/types"
27+
"k8s.io/client-go/tools/cache"
28+
"sigs.k8s.io/controller-runtime/pkg/client"
29+
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
30+
k8s "sigs.k8s.io/vsphere-csi-driver/v3/pkg/kubernetes"
31+
32+
"sigs.k8s.io/vsphere-csi-driver/v3/pkg/csi/service/logger"
33+
"sigs.k8s.io/vsphere-csi-driver/v3/pkg/internalapis"
34+
"sigs.k8s.io/vsphere-csi-driver/v3/pkg/internalapis/cnsoperator/cnsvolumeattachment/v1alpha1"
35+
cnsoperatortypes "sigs.k8s.io/vsphere-csi-driver/v3/pkg/syncer/cnsoperator/types"
36+
)
37+
38+
var (
39+
cnsVolumeAttachmentInstanceLock sync.Mutex
40+
cnsVolumeAttachmentInstance *cnsVolumeAttachment
41+
)
42+
43+
// CnsVolumeAttachment exposes an interface to support adding
44+
// and removing information about attached VMs to a PVC.
45+
type CnsVolumeAttachment interface {
46+
// Add adds the input VM UUID to the list of
47+
// attached VMs for the given volume.
48+
AddVmToAttachedList(ctx context.Context, volumeName, VmUUID string) error
49+
// RemoveVmFromAttachedList removes the input VM UUID from
50+
// the list of attached VMs for the given volume.
51+
RemoveVmFromAttachedList(ctx context.Context, volumeName, VmUUID string) error
52+
// CnsVolumeAttachmentExistsForPvc returns true if CnsVolumeAttachment for a PVC is found.
53+
CnsVolumeAttachmentExistsForPvc(ctx context.Context, volumeName string) (bool, error)
54+
}
55+
56+
// cnsVolumeAttachment maintains a client to the API
57+
// server for operations on CnsVolumeAttachment instance.
58+
// It also contains a per instance lock to handle
59+
// concurrent operations.
60+
type cnsVolumeAttachment struct {
61+
client client.Client
62+
// Per volume lock for concurrent access to CnsVolumeAttachment instances.
63+
// Keys are strings representing PVC names.
64+
// Values are individual sync.Mutex locks that need to be held
65+
// to make updates to the CnsVolumeAttachment instance on the API server.
66+
volumeLock *sync.Map
67+
}
68+
69+
// GetCnsVolumeAttachmentInstance returns a singleton of type CnsVolumeAttachment.
70+
// Initializes the singleton if not already initialized.
71+
func GetCnsVolumeAttachmentInstance(ctx context.Context) (CnsVolumeAttachment, error) {
72+
cnsVolumeAttachmentInstanceLock.Lock()
73+
defer cnsVolumeAttachmentInstanceLock.Unlock()
74+
if cnsVolumeAttachmentInstance == nil {
75+
log := logger.GetLogger(ctx)
76+
config, err := k8s.GetKubeConfig(ctx)
77+
if err != nil {
78+
log.Errorf("failed to get kubeconfig. Err: %v", err)
79+
return nil, err
80+
}
81+
k8sclient, err := k8s.NewClientForGroup(ctx, config, internalapis.GroupName)
82+
if err != nil {
83+
log.Errorf("failed to create k8s client. Err: %v", err)
84+
return nil, err
85+
}
86+
cnsVolumeAttachmentInstance = &cnsVolumeAttachment{
87+
client: k8sclient,
88+
volumeLock: &sync.Map{},
89+
}
90+
}
91+
92+
return cnsVolumeAttachmentInstance, nil
93+
}
94+
95+
// Add adds the input VM UUID to the list of
96+
// attached VMs for the given volume.
97+
// Callers need to specify cnsVolumeAttachment as a combination of
98+
// "<SV-namespace>/<SV-PVC-name>". This combination is used to uniquely
99+
// identify CnsVolumeAttachment instances.
100+
// The instance is created if it doesn't exist.
101+
// Returns an error if the operation cannot be persisted on the API server.
102+
func (f *cnsVolumeAttachment) AddVmToAttachedList(ctx context.Context,
103+
volumeName, VmUUID string) error {
104+
log := logger.GetLogger(ctx)
105+
106+
log.Infof("Adding VM %s to cnsVolumeAttachment %s",
107+
VmUUID, volumeName)
108+
actual, _ := f.volumeLock.LoadOrStore(volumeName, &sync.Mutex{})
109+
instanceLock, ok := actual.(*sync.Mutex)
110+
if !ok {
111+
return fmt.Errorf("failed to cast lock for cnsVolumeAttachment instance: %s", volumeName)
112+
}
113+
instanceLock.Lock()
114+
defer instanceLock.Unlock()
115+
116+
instance := &v1alpha1.CnsVolumeAttachment{}
117+
instanceNamespace, instanceName, err := cache.SplitMetaNamespaceKey(volumeName)
118+
if err != nil {
119+
log.Errorf("failed to split key %s with error: %+v", volumeName, err)
120+
return err
121+
}
122+
instanceKey := types.NamespacedName{
123+
Namespace: instanceNamespace,
124+
Name: instanceName,
125+
}
126+
err = f.client.Get(ctx, instanceKey, instance)
127+
if err != nil {
128+
if errors.IsNotFound(err) {
129+
// Create the instance as it does not exist on the API server.
130+
instance = &v1alpha1.CnsVolumeAttachment{
131+
ObjectMeta: v1.ObjectMeta{
132+
Name: instanceName,
133+
Namespace: instanceNamespace,
134+
// Add finalizer so that CnsVolumeAttachment instance doesn't get deleted abruptly
135+
Finalizers: []string{cnsoperatortypes.CNSFinalizer},
136+
},
137+
Spec: v1alpha1.CnsVolumeAttachmentSpec{
138+
AttachedVms: []string{
139+
VmUUID,
140+
},
141+
},
142+
}
143+
log.Debugf("Creating cnsVolumeAttachment instance %s with spec: %+v", volumeName, instance)
144+
err = f.client.Create(ctx, instance)
145+
if err != nil {
146+
log.Errorf("failed to create cnsVolumeAttachment instance %s with error: %+v", volumeName, err)
147+
return err
148+
}
149+
return nil
150+
}
151+
log.Errorf("failed to get cnsVolumeAttachment instance %s with error: %+v", volumeName, err)
152+
return err
153+
}
154+
155+
// Verify if input VmUUID exists in existing AttachedVMs list.
156+
log.Debugf("Verifying if VM %s exists in current list of attached Vms. Current list: %+v",
157+
VmUUID, instance.Spec.AttachedVms)
158+
oldAttachedVmsList := instance.Spec.AttachedVms
159+
for _, oldAttachedVM := range oldAttachedVmsList {
160+
if oldAttachedVM == VmUUID {
161+
log.Debugf("Found VM %s in list. Returning.", VmUUID)
162+
return nil
163+
}
164+
}
165+
newAttachVmsList := append(oldAttachedVmsList, VmUUID)
166+
instance.Spec.AttachedVms = newAttachVmsList
167+
log.Debugf("Updating cnsVolumeAttachment instance %s with spec: %+v", volumeName, instance)
168+
err = f.client.Update(ctx, instance)
169+
if err != nil {
170+
log.Errorf("failed to update cnsVolumeAttachment instance %s/%s with error: %+v", volumeName, err)
171+
}
172+
return err
173+
}
174+
175+
// RemoveVmFromAttachedList removes the input VM UUID from
176+
// the list of attached VMs for the given volume.
177+
// Callers need to specify volumeName as a combination of
178+
// "<SV-namespace>/<SV-PVC-name>". This combination is used to uniquely
179+
// identify CnsVolumeAttachment instances.
180+
// If the given VM was the last client for this file volume, the instance is
181+
// deleted from the API server.
182+
// Returns an error if the operation cannot be persisted on the API server.
183+
func (f *cnsVolumeAttachment) RemoveVmFromAttachedList(ctx context.Context,
184+
volumeName, VmUUID string) error {
185+
log := logger.GetLogger(ctx)
186+
log.Infof("Removing VmUUID %s from cnsVolumeAttachment %s",
187+
VmUUID, volumeName)
188+
actual, _ := f.volumeLock.LoadOrStore(volumeName, &sync.Mutex{})
189+
instanceLock, ok := actual.(*sync.Mutex)
190+
if !ok {
191+
return fmt.Errorf("failed to cast lock for cnsVolumeAttachment instance: %s", volumeName)
192+
}
193+
instanceLock.Lock()
194+
defer instanceLock.Unlock()
195+
instance := &v1alpha1.CnsVolumeAttachment{}
196+
instanceNamespace, instanceName, err := cache.SplitMetaNamespaceKey(volumeName)
197+
if err != nil {
198+
log.Errorf("failed to split key %s with error: %+v", volumeName, err)
199+
return err
200+
}
201+
instanceKey := types.NamespacedName{
202+
Namespace: instanceNamespace,
203+
Name: instanceName,
204+
}
205+
err = f.client.Get(ctx, instanceKey, instance)
206+
if err != nil {
207+
if errors.IsNotFound(err) {
208+
log.Infof("cnsVolumeAttachment instance %s does not exist on API server", volumeName)
209+
return nil
210+
}
211+
log.Errorf("failed to get cnsVolumeAttachment instance %s with error: %+v", volumeName, err)
212+
return err
213+
}
214+
215+
log.Debugf("Verifying if VM UUID %s exists in list of already attached VMs. Current list: %+v",
216+
volumeName, instance.Spec.AttachedVms)
217+
for index, existingAttachedtVM := range instance.Spec.AttachedVms {
218+
if volumeName == existingAttachedtVM {
219+
log.Debugf("Removing VmUUID %s from Attached VMs list", VmUUID)
220+
instance.Spec.AttachedVms = append(
221+
instance.Spec.AttachedVms[:index],
222+
instance.Spec.AttachedVms[index+1:]...)
223+
if len(instance.Spec.AttachedVms) == 0 {
224+
log.Infof("Deleting cnsVolumeAttachment instance %s from API server", volumeName)
225+
// Remove finalizer from CnsVolumeAttachment instance
226+
err = removeFinalizer(ctx, f.client, instance)
227+
if err != nil {
228+
log.Errorf("failed to remove finalizer from cnsVolumeAttachment instance %s with error: %+v",
229+
volumeName, err)
230+
}
231+
err = f.client.Delete(ctx, instance)
232+
if err != nil {
233+
// In case of namespace deletion, we will have deletion timestamp added on the
234+
// CnsVolumeAttachment instance. So, as soon as we delete finalizer, instance might
235+
// get deleted immediately. In such cases we will get NotFound error here, return success
236+
// if instance is already deleted.
237+
if errors.IsNotFound(err) {
238+
log.Infof("cnsVolumeAttachment instance %s seems to be already deleted.", volumeName)
239+
f.volumeLock.Delete(volumeName)
240+
return nil
241+
}
242+
log.Errorf("failed to delete cnsVolumeAttachment instance %s with error: %+v", volumeName, err)
243+
return err
244+
}
245+
f.volumeLock.Delete(volumeName)
246+
return nil
247+
}
248+
log.Debugf("Updating cnsVolumeAttachment instance %s with spec: %+v", volumeName, instance)
249+
err = f.client.Update(ctx, instance)
250+
if err != nil {
251+
log.Errorf("failed to update cnsVolumeAttachment instance %s with error: %+v", volumeName, err)
252+
}
253+
return err
254+
}
255+
}
256+
log.Debugf("Could not find VM %s in list. Returning.", VmUUID)
257+
return nil
258+
}
259+
260+
// CnsVolumeAttachmentExistsForPvc returns true if CnsVolumeAttachment instance
261+
// for the given PVC exists.
262+
// Callers need to specify volumeName as a combination of
263+
// "<SV-namespace>/<SV-PVC-name>". This combination is used to uniquely
264+
// identify CnsVolumeAttachment instances.
265+
// If the given VM was the last client for this file volume, the instance is
266+
// deleted from the API server.
267+
// Returns an error if the operation cannot be persisted on the API server.
268+
func (f *cnsVolumeAttachment) CnsVolumeAttachmentExistsForPvc(ctx context.Context, volumeName string) (bool, error) {
269+
log := logger.GetLogger(ctx)
270+
271+
log.Infof("Fetching cnsfilevolumeclient instance for volume %s", volumeName)
272+
actual, _ := f.volumeLock.LoadOrStore(volumeName, &sync.Mutex{})
273+
instanceLock, ok := actual.(*sync.Mutex)
274+
if !ok {
275+
return true, fmt.Errorf("failed to cast lock for cnsVolumeAttachment instance: %s", volumeName)
276+
}
277+
instanceLock.Lock()
278+
defer instanceLock.Unlock()
279+
280+
instance := &v1alpha1.CnsVolumeAttachment{}
281+
instanceNamespace, instanceName, err := cache.SplitMetaNamespaceKey(volumeName)
282+
if err != nil {
283+
log.Errorf("failed to split key %s with error: %+v", volumeName, err)
284+
return true, err
285+
}
286+
instanceKey := types.NamespacedName{
287+
Namespace: instanceNamespace,
288+
Name: instanceName,
289+
}
290+
err = f.client.Get(ctx, instanceKey, instance)
291+
if err != nil {
292+
if errors.IsNotFound(err) {
293+
// CnsVolumeAttachment instance not found.
294+
// This means PVC is not being used by any VM.
295+
return false, nil
296+
}
297+
log.Errorf("failed to get cnsVolumeAttachment instance %s with error: %+v", volumeName, err)
298+
return true, err
299+
}
300+
return true, nil
301+
}
302+
303+
// removeFinalizer will remove the CNS Finalizer = cns.vmware.com,
304+
// from a given CnsVolumeAttachment instance.
305+
func removeFinalizer(ctx context.Context, client client.Client,
306+
instance *v1alpha1.CnsVolumeAttachment) error {
307+
if !controllerutil.ContainsFinalizer(instance, cnsoperatortypes.CNSFinalizer) {
308+
// Finalizer not present on instance. Nothing to do.
309+
return nil
310+
}
311+
312+
if !controllerutil.AddFinalizer(instance, cnsoperatortypes.CNSFinalizer) {
313+
return fmt.Errorf("failed to add CNS finalizer %s to CnsVolumeAttachment "+
314+
"instance %s in namespace %s", cnsoperatortypes.CNSFinalizer, instance.Name,
315+
instance.Namespace)
316+
}
317+
318+
err := client.Update(ctx, instance)
319+
if err != nil {
320+
return fmt.Errorf("failed to update finalizer CnsVolumeAttachment instance with name: %q on namespace: %q",
321+
instance.Name, instance.Namespace)
322+
}
323+
324+
return nil
325+
}

‎pkg/internalapis/cnsoperator/cnsvolumeattachment/v1alpha1/cnsvolumeattachment_types.go

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
/*
2+
Copyright 2025 The Kubernetes authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package v1alpha1
18+
19+
import (
20+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
21+
)
22+
23+
// CnsVolumeAttachmentSpec contains the list of VM UUIDs attached
24+
// to the given volume
25+
type CnsVolumeAttachmentSpec struct {
26+
AttachedVms []string `json:"attachedVms,omitempty"`
27+
}
28+
29+
// +kubebuilder:object:root=true
30+
// +kubebuilder:subresource:status
31+
32+
// CnsVolumeAttachment is the Schema for the cnsvolumeattachment CRD. This CRD is used by
33+
// CNS-CSI for internal bookkeeping purposes only and is not an API.
34+
type CnsVolumeAttachment struct {
35+
metav1.TypeMeta `json:",inline"`
36+
metav1.ObjectMeta `json:"metadata,omitempty"`
37+
38+
Spec CnsVolumeAttachmentSpec `json:"spec,omitempty"`
39+
}
40+
41+
// +kubebuilder:object:root=true
42+
43+
// CnsVolumeAttachmentList contains a list of CnsVolumeAttachment
44+
type CnsVolumeAttachmentList struct {
45+
metav1.TypeMeta `json:",inline"`
46+
metav1.ListMeta `json:"metadata,omitempty"`
47+
Items []CnsVolumeAttachment `json:"items"`
48+
}

‎pkg/internalapis/cnsoperator/cnsvolumeattachment/v1alpha1/doc.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
// +k8s:deepcopy-gen=package
2+
// +k8s:defaulter-gen=TypeMeta
3+
// +groupName=cns.vmware.com
4+
5+
package v1alpha1

0 commit comments

Comments
 (0)