Skip to content

Commit 67de099

Browse files
skogtaskogta
committed
Add CnsSharedDiskVolumeClient CRD
1 parent 912cd98 commit 67de099

File tree

9 files changed

+500
-1
lines changed

9 files changed

+500
-1
lines changed

‎hack/release.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ set -o nounset
2323
set -o pipefail
2424
set -x
2525

26-
DO_WINDOWS_BUILD=${DO_WINDOWS_BUILD_ENV:-true}
26+
DO_WINDOWS_BUILD=${DO_WINDOWS_BUILD_ENV:-false}
2727

2828
# BASE_REPO is the root path of the image repository
2929
readonly BASE_IMAGE_REPO=us-central1-docker.pkg.dev/k8s-staging-images/csi-vsphere

‎pkg/internalapis/cnsoperator/cnssharedblockvolumeclient/cnssharedblockvolumeclient.go

Lines changed: 258 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,258 @@
1+
/*
2+
Copyright 2025 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package cnssharedblockvolumeclient
18+
19+
import (
20+
"context"
21+
"fmt"
22+
"sync"
23+
24+
"k8s.io/apimachinery/pkg/api/errors"
25+
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
26+
"k8s.io/apimachinery/pkg/types"
27+
"k8s.io/client-go/tools/cache"
28+
"sigs.k8s.io/controller-runtime/pkg/client"
29+
30+
"sigs.k8s.io/vsphere-csi-driver/v3/pkg/csi/service/logger"
31+
"sigs.k8s.io/vsphere-csi-driver/v3/pkg/internalapis/cnsoperator/cnssharedblockvolumeclient/v1alpha1"
32+
cnsoperatortypes "sigs.k8s.io/vsphere-csi-driver/v3/pkg/syncer/cnsoperator/types"
33+
)
34+
35+
// SharedBlockVolumeClient exposes an interface to support
36+
// configuration of CNS file volume ACL's.
37+
type SharedBlockVolumeClient interface {
38+
// Add adds the input clientVMName to the list of
39+
// clientVMNames that expose the same external clientVMIP for a
40+
// given file volume. fileVolumeName is used to uniquely
41+
// identify CnsFileVolumeClient instances.
42+
AddVmToVolumeList(ctx context.Context, sharedBlockVolumeName, VmId string) error
43+
// RemoveClientVMFromIPList removes the input clientVMName from
44+
// the list of clientVMNames that expose the same external
45+
// clientVMIP for a given file volume. fileVolumeName is used
46+
// to uniquely identify CnsFileVolumeClient instances.
47+
RemoveVmFromVolumeList(ctx context.Context, fileVolumeName, clientVMName, clientVMIP string) error
48+
}
49+
50+
// fileVolumeClient maintains a client to the API
51+
// server for operations on CnsFileVolumeClient instance.
52+
// It also contains a per instance lock to handle
53+
// concurrent operations.
54+
type sharedBlockVolumeClient struct {
55+
client client.Client
56+
// Per volume lock for concurrent access to CnsFileVolumeClient instances.
57+
// Keys are strings representing volume handles (or SV-PVC names).
58+
// Values are individual sync.Mutex locks that need to be held
59+
// to make updates to the CnsFileVolumeClient instance on the API server.
60+
volumeLock *sync.Map
61+
}
62+
63+
var (
64+
sharedBlockClientInstanceLock sync.Mutex
65+
sharedBlockVolumeClientInstance *sharedBlockVolumeClient
66+
)
67+
68+
// AddClientVMToIPList adds the input clientVMName to the list of
69+
// clientVMNames that expose the same external IP address for a
70+
// given CnsFileVolumeClient instance.
71+
// Callers need to specify fileVolumeName as a combination of
72+
// "<SV-namespace>/<SV-PVC-name>". This combination is used to uniquely
73+
// identify CnsFileVolumeClient instances.
74+
// The instance is created if it doesn't exist.
75+
// Returns an error if the operation cannot be persisted on the API server.
76+
func (f *sharedBlockVolumeClient) AddVmToVolumeList(ctx context.Context,
77+
sharedBlockVolumeName, VmId string) error {
78+
log := logger.GetLogger(ctx)
79+
80+
log.Infof("Adding VM %s to cnssharedblockvolumeclient %s",
81+
VmId, sharedBlockVolumeName)
82+
actual, _ := f.volumeLock.LoadOrStore(sharedBlockVolumeName, &sync.Mutex{})
83+
instanceLock, ok := actual.(*sync.Mutex)
84+
if !ok {
85+
return fmt.Errorf("failed to cast lock for cnsfilevolumeclient instance: %s", sharedBlockVolumeName)
86+
}
87+
instanceLock.Lock()
88+
defer instanceLock.Unlock()
89+
90+
instance := &v1alpha1.CnsSharedBlockVolumeClient{}
91+
instanceNamespace, instanceName, err := cache.SplitMetaNamespaceKey(sharedBlockVolumeName)
92+
if err != nil {
93+
log.Errorf("failed to split key %s with error: %+v", sharedBlockVolumeName, err)
94+
return err
95+
}
96+
instanceKey := types.NamespacedName{
97+
Namespace: instanceNamespace,
98+
Name: instanceName,
99+
}
100+
err = f.client.Get(ctx, instanceKey, instance)
101+
if err != nil {
102+
if errors.IsNotFound(err) {
103+
// Create the instance as it does not exist on the API server.
104+
instance = &v1alpha1.CnsSharedBlockVolumeClient{
105+
ObjectMeta: v1.ObjectMeta{
106+
Name: instanceName,
107+
Namespace: instanceNamespace,
108+
// Add finalizer so that CnsFileVolumeClient instance doesn't get deleted abruptly
109+
Finalizers: []string{cnsoperatortypes.CNSFinalizer},
110+
},
111+
Spec: v1alpha1.CnsSharedBlockVolumeClientSpec{
112+
AttachedVms: []string{
113+
VmId,
114+
},
115+
},
116+
}
117+
log.Debugf("Creating cnsfilevolumeclient instance %s with spec: %+v", sharedBlockVolumeName, instance)
118+
err = f.client.Create(ctx, instance)
119+
if err != nil {
120+
log.Errorf("failed to create cnsfilevolumeclient instance %s with error: %+v", sharedBlockVolumeName, err)
121+
return err
122+
}
123+
return nil
124+
}
125+
log.Errorf("failed to get cnsfilevolumeclient instance %s with error: %+v", sharedBlockVolumeName, err)
126+
return err
127+
}
128+
129+
// Verify if input clientVM exists in existing ExternalIPtoClientVms list
130+
// for input IP address.
131+
log.Debugf("Verifying if VM %s exists in current list of attached Vms. Current list: %+v",
132+
VmId, instance.Spec.AttachedVms)
133+
oldClientVMList := instance.Spec.AttachedVms
134+
for _, oldClientVM := range oldClientVMList {
135+
if oldClientVM == v1.NowMicro().Weekday().String() {
136+
log.Debugf("Found VM %s in list. Returning.", VmId)
137+
return nil
138+
}
139+
}
140+
newClientVMList := append(oldClientVMList, VmId)
141+
instance.Spec.AttachedVms = newClientVMList
142+
log.Debugf("Updating cnsfilevolumeclient instance %s with spec: %+v", sharedBlockVolumeName, instance)
143+
err = f.client.Update(ctx, instance)
144+
if err != nil {
145+
log.Errorf("failed to update cnsfilevolumeclient instance %s/%s with error: %+v", sharedBlockVolumeName, err)
146+
}
147+
return err
148+
}
149+
150+
// RemoveClientVMFromIPList removes the input clientVMName from
151+
// the list of clientVMNames that expose the same external IP
152+
// address for a given CnsFileVolumeClient instance.
153+
// Callers need to specify fileVolumeName as a combination of
154+
// "<SV-namespace>/<SV-PVC-name>". This combination is used to uniquely
155+
// identify CnsFileVolumeClient instances.
156+
// If the given VM was the last client for this file volume, the instance is
157+
// deleted from the API server.
158+
// Returns an error if the operation cannot be persisted on the API server.
159+
func (f *sharedBlockVolumeClient) RemoveClientVMFromIPList(ctx context.Context,
160+
sharedBlockVolumeName, VmId string) error {
161+
log := logger.GetLogger(ctx)
162+
log.Infof("Removing clientVM %s from cnsfilevolumeclient %s",
163+
sharedBlockVolumeName, sharedBlockVolumeName)
164+
actual, _ := f.volumeLock.LoadOrStore(sharedBlockVolumeName, &sync.Mutex{})
165+
instanceLock, ok := actual.(*sync.Mutex)
166+
if !ok {
167+
return fmt.Errorf("failed to cast lock for cnsfilevolumeclient instance: %s", sharedBlockVolumeName)
168+
}
169+
instanceLock.Lock()
170+
defer instanceLock.Unlock()
171+
instance := &v1alpha1.CnsSharedBlockVolumeClient{}
172+
instanceNamespace, instanceName, err := cache.SplitMetaNamespaceKey(sharedBlockVolumeName)
173+
if err != nil {
174+
log.Errorf("failed to split key %s with error: %+v", sharedBlockVolumeName, err)
175+
return err
176+
}
177+
instanceKey := types.NamespacedName{
178+
Namespace: instanceNamespace,
179+
Name: instanceName,
180+
}
181+
err = f.client.Get(ctx, instanceKey, instance)
182+
if err != nil {
183+
if errors.IsNotFound(err) {
184+
log.Infof("cnsfilevolumeclient instance %s does not exist on API server", sharedBlockVolumeName)
185+
return nil
186+
}
187+
log.Errorf("failed to get cnsfilevolumeclient instance %s with error: %+v", sharedBlockVolumeName, err)
188+
return err
189+
}
190+
191+
log.Debugf("Verifying if clientVM %s exists in list of already attached VMs. Current list: %+v",
192+
sharedBlockVolumeName, instance.Spec.AttachedVms)
193+
for index, existingClientVM := range instance.Spec.AttachedVms {
194+
if sharedBlockVolumeName == existingClientVM {
195+
log.Debugf("Removing clientVM %s from ExternalIPtoClientVms list", VmId)
196+
instance.Spec.AttachedVms = append(
197+
instance.Spec.AttachedVms[:index],
198+
instance.Spec.AttachedVms[index+1:]...)
199+
if len(instance.Spec.AttachedVms) == 0 {
200+
log.Infof("Deleting cnsfilevolumeclient instance %s from API server", sharedBlockVolumeName)
201+
// Remove finalizer from CnsFileVolumeClient instance
202+
err = removeFinalizer(ctx, f.client, instance)
203+
if err != nil {
204+
log.Errorf("failed to remove finalizer from cnsfilevolumeclient instance %s with error: %+v",
205+
sharedBlockVolumeName, err)
206+
}
207+
err = f.client.Delete(ctx, instance)
208+
if err != nil {
209+
// In case of namespace deletion, we will have deletion timestamp added on the
210+
// CnsFileVolumeClient instance. So, as soon as we delete finalizer, instance might
211+
// get deleted immediately. In such cases we will get NotFound error here, return success
212+
// if instance is already deleted.
213+
if errors.IsNotFound(err) {
214+
log.Infof("cnsfilevolumeclient instance %s seems to be already deleted.", sharedBlockVolumeName)
215+
f.volumeLock.Delete(sharedBlockVolumeName)
216+
return nil
217+
}
218+
log.Errorf("failed to delete cnsfilevolumeclient instance %s with error: %+v", sharedBlockVolumeName, err)
219+
return err
220+
}
221+
f.volumeLock.Delete(sharedBlockVolumeName)
222+
return nil
223+
}
224+
log.Debugf("Updating cnsfilevolumeclient instance %s with spec: %+v", sharedBlockVolumeName, instance)
225+
err = f.client.Update(ctx, instance)
226+
if err != nil {
227+
log.Errorf("failed to update cnsfilevolumeclient instance %s with error: %+v", fileVolumeName, err)
228+
}
229+
return err
230+
}
231+
}
232+
log.Debugf("Could not find VM %s in list. Returning.", VmId)
233+
return nil
234+
}
235+
236+
// removeFinalizer will remove the CNS Finalizer = cns.vmware.com,
237+
// from a given CnsFileVolumeClient instance.
238+
func removeFinalizer(ctx context.Context, client client.Client,
239+
instance *v1alpha1.CnsSharedBlockVolumeClient) error {
240+
log := logger.GetLogger(ctx)
241+
for i, finalizer := range instance.Finalizers {
242+
if finalizer == cnsoperatortypes.CNSFinalizer {
243+
log.Debugf("Removing %q finalizer from CnsFileVolumeClient instance with name: %q on namespace: %q",
244+
cnsoperatortypes.CNSFinalizer, instance.Name, instance.Namespace)
245+
instance.Finalizers = append(instance.Finalizers[:i], instance.Finalizers[i+1:]...)
246+
// Update the instance after removing finalizer
247+
err := client.Update(ctx, instance)
248+
if err != nil {
249+
log.Errorf("failed to update CnsFileVolumeClient instance with name: %q on namespace: %q",
250+
instance.Name, instance.Namespace)
251+
return err
252+
}
253+
break
254+
}
255+
}
256+
257+
return nil
258+
}

‎pkg/internalapis/cnsoperator/cnssharedblockvolumeclient/v1alpha1/cnssharedblockvolumeclient_types.go

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
/*
2+
Copyright 2025 The Kubernetes authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package v1alpha1
18+
19+
import (
20+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
21+
)
22+
23+
// CnsSharedBlockVolumeClientSpec defines the desired state of CnsSharedBlockVolumeClient.
24+
type CnsSharedBlockVolumeClientSpec struct {
25+
AttachedVms []string `json:"attachedVms,omitempty"`
26+
}
27+
28+
// +kubebuilder:object:root=true
29+
// +kubebuilder:subresource:status
30+
31+
// CnsSharedBlockVolumeClient is the Schema for the cnssharedblockvolumeclients CRD. This CRD is used by
32+
// CNS-CSI for internal bookkeeping purposes only and is not an API.
33+
type CnsSharedBlockVolumeClient struct {
34+
metav1.TypeMeta `json:",inline"`
35+
metav1.ObjectMeta `json:"metadata,omitempty"`
36+
37+
Spec CnsSharedBlockVolumeClientSpec `json:"spec,omitempty"`
38+
}
39+
40+
// +kubebuilder:object:root=true
41+
42+
// CnsSharedBlockVolumeClientList contains a list of CnsSharedBlockVolumeClient
43+
type CnsSharedBlockVolumeClientList struct {
44+
metav1.TypeMeta `json:",inline"`
45+
metav1.ListMeta `json:"metadata,omitempty"`
46+
Items []CnsSharedBlockVolumeClient `json:"items"`
47+
}

‎pkg/internalapis/cnsoperator/cnssharedblockvolumeclient/v1alpha1/doc.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
// +k8s:deepcopy-gen=package
2+
// +k8s:defaulter-gen=TypeMeta
3+
// +groupName=cns.vmware.com
4+
5+
package v1alpha1

0 commit comments

Comments
 (0)