Skip to content

Commit 553444d

Browse files
andyzhangxandyzhangx
authored
Merge pull request #427 from andyzhangx/nfs-e2e-test
test: add nfs e2e test
2 parents bcaecd2 + 509e37d commit 553444d

File tree

7 files changed

+233
-8
lines changed

7 files changed

+233
-8
lines changed

‎pkg/blob/azure.go

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@ import (
2424
"golang.org/x/net/context"
2525

2626
kv "github.com/Azure/azure-sdk-for-go/services/keyvault/2016-10-01/keyvault"
27+
"github.com/Azure/azure-sdk-for-go/services/network/mgmt/2020-08-01/network"
28+
2729
"github.com/Azure/go-autorest/autorest"
2830
"github.com/Azure/go-autorest/autorest/adal"
2931
"github.com/Azure/go-autorest/autorest/azure"
@@ -39,6 +41,7 @@ import (
3941
var (
4042
DefaultAzureCredentialFileEnv = "AZURE_CREDENTIAL_FILE"
4143
DefaultCredFilePath = "/etc/kubernetes/azure.json"
44+
storageService = "Microsoft.Storage"
4245
)
4346

4447
// IsAzureStackCloud decides whether the driver is running on Azure Stack Cloud.
@@ -163,6 +166,62 @@ func (d *Driver) getServicePrincipalToken(env azure.Environment, resource string
163166
resource)
164167
}
165168

169+
func (d *Driver) updateSubnetServiceEndpoints(ctx context.Context) error {
170+
resourceGroup := d.cloud.ResourceGroup
171+
if len(d.cloud.VnetResourceGroup) > 0 {
172+
resourceGroup = d.cloud.VnetResourceGroup
173+
}
174+
location := d.cloud.Location
175+
vnetName := d.cloud.VnetName
176+
subnetName := d.cloud.SubnetName
177+
178+
if d.cloud.SubnetsClient == nil {
179+
return fmt.Errorf("SubnetsClient is nil")
180+
}
181+
182+
subnet, err := d.cloud.SubnetsClient.Get(ctx, resourceGroup, vnetName, subnetName, "")
183+
if err != nil {
184+
return fmt.Errorf("failed to get the subnet %s under vnet %s: %v", subnetName, vnetName, err)
185+
}
186+
endpointLocaions := []string{location}
187+
storageServiceEndpoint := network.ServiceEndpointPropertiesFormat{
188+
Service: &storageService,
189+
Locations: &endpointLocaions,
190+
}
191+
storageServiceExists := false
192+
if subnet.SubnetPropertiesFormat == nil {
193+
subnet.SubnetPropertiesFormat = &network.SubnetPropertiesFormat{}
194+
}
195+
if subnet.SubnetPropertiesFormat.ServiceEndpoints == nil {
196+
subnet.SubnetPropertiesFormat.ServiceEndpoints = &[]network.ServiceEndpointPropertiesFormat{}
197+
}
198+
serviceEndpoints := *subnet.SubnetPropertiesFormat.ServiceEndpoints
199+
for _, v := range serviceEndpoints {
200+
if v.Service != nil && *v.Service == storageService {
201+
storageServiceExists = true
202+
klog.V(4).Infof("serviceEndpoint(%s) is already in subnet(%s)", storageService, subnetName)
203+
break
204+
}
205+
}
206+
207+
if !storageServiceExists {
208+
serviceEndpoints = append(serviceEndpoints, storageServiceEndpoint)
209+
subnet.SubnetPropertiesFormat.ServiceEndpoints = &serviceEndpoints
210+
211+
lockKey := resourceGroup + vnetName + subnetName
212+
d.subnetLockMap.LockEntry(lockKey)
213+
defer d.subnetLockMap.UnlockEntry(lockKey)
214+
215+
err = d.cloud.SubnetsClient.CreateOrUpdate(context.Background(), resourceGroup, vnetName, subnetName, subnet)
216+
if err != nil {
217+
return fmt.Errorf("failed to update the subnet %s under vnet %s: %v", subnetName, vnetName, err)
218+
}
219+
klog.V(4).Infof("serviceEndpoint(%s) is appended in subnet(%s)", storageService, subnetName)
220+
}
221+
222+
return nil
223+
}
224+
166225
func getKubeClient(kubeconfig string) (*kubernetes.Clientset, error) {
167226
var (
168227
config *rest.Config

‎pkg/blob/azure_test.go

Lines changed: 131 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -24,10 +24,16 @@ import (
2424
"reflect"
2525
"testing"
2626

27+
"github.com/Azure/azure-sdk-for-go/services/network/mgmt/2020-08-01/network"
28+
"github.com/golang/mock/gomock"
29+
2730
"github.com/Azure/go-autorest/autorest/azure"
2831
"github.com/stretchr/testify/assert"
2932

30-
azure2 "sigs.k8s.io/cloud-provider-azure/pkg/provider"
33+
"sigs.k8s.io/cloud-provider-azure/pkg/azureclients/subnetclient/mocksubnetclient"
34+
azureprovider "sigs.k8s.io/cloud-provider-azure/pkg/provider"
35+
36+
"sigs.k8s.io/cloud-provider-azure/pkg/retry"
3137
)
3238

3339
// TestGetCloudProvider tests the func getCloudProvider().
@@ -160,7 +166,7 @@ func TestGetServicePrincipalToken(t *testing.T) {
160166
}
161167
resource := "unit-test"
162168
d := NewFakeDriver()
163-
d.cloud = &azure2.Cloud{}
169+
d.cloud = &azureprovider.Cloud{}
164170
_, err := d.getServicePrincipalToken(env, resource)
165171
expectedErr := fmt.Errorf("parameter 'clientID' cannot be empty")
166172
if !reflect.DeepEqual(expectedErr, err) {
@@ -178,7 +184,7 @@ func TestGetKeyvaultToken(t *testing.T) {
178184
KeyVaultEndpoint: "unit-test",
179185
}
180186
d := NewFakeDriver()
181-
d.cloud = &azure2.Cloud{}
187+
d.cloud = &azureprovider.Cloud{}
182188
d.cloud.Environment = env
183189
_, err := d.getKeyvaultToken()
184190
expectedErr := fmt.Errorf("parameter 'clientID' cannot be empty")
@@ -198,7 +204,7 @@ func TestInitializeKvClient(t *testing.T) {
198204
KeyVaultEndpoint: "unit-test",
199205
}
200206
d := NewFakeDriver()
201-
d.cloud = &azure2.Cloud{}
207+
d.cloud = &azureprovider.Cloud{}
202208
d.cloud.Environment = env
203209
_, err := d.initializeKvClient()
204210
expectedErr := fmt.Errorf("parameter 'clientID' cannot be empty")
@@ -217,7 +223,7 @@ func TestGetKeyVaultSecretContent(t *testing.T) {
217223
KeyVaultEndpoint: "unit-test",
218224
}
219225
d := NewFakeDriver()
220-
d.cloud = &azure2.Cloud{}
226+
d.cloud = &azureprovider.Cloud{}
221227
d.cloud.Environment = env
222228
valueURL := "unit-test"
223229
secretName := "unit-test"
@@ -245,3 +251,123 @@ func createTestFile(path string) error {
245251

246252
return nil
247253
}
254+
255+
func TestUpdateSubnetServiceEndpoints(t *testing.T) {
256+
d := NewFakeDriver()
257+
ctrl := gomock.NewController(t)
258+
defer ctrl.Finish()
259+
mockSubnetClient := mocksubnetclient.NewMockInterface(ctrl)
260+
261+
config := azureprovider.Config{
262+
ResourceGroup: "rg",
263+
Location: "loc",
264+
VnetName: "fake-vnet",
265+
SubnetName: "fake-subnet",
266+
}
267+
268+
d.cloud = &azureprovider.Cloud{
269+
SubnetsClient: mockSubnetClient,
270+
Config: config,
271+
}
272+
ctx := context.TODO()
273+
274+
testCases := []struct {
275+
name string
276+
testFunc func(t *testing.T)
277+
}{
278+
{
279+
name: "[fail] no subnet",
280+
testFunc: func(t *testing.T) {
281+
retErr := retry.NewError(false, fmt.Errorf("the subnet does not exist"))
282+
mockSubnetClient.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(network.Subnet{}, retErr).Times(1)
283+
expectedErr := fmt.Errorf("failed to get the subnet %s under vnet %s: %v", config.SubnetName, config.VnetName, retErr)
284+
err := d.updateSubnetServiceEndpoints(ctx)
285+
if !reflect.DeepEqual(err, expectedErr) {
286+
t.Errorf("Unexpected error: %v", err)
287+
}
288+
},
289+
},
290+
{
291+
name: "[success] subnetPropertiesFormat is nil",
292+
testFunc: func(t *testing.T) {
293+
mockSubnetClient.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(network.Subnet{}, nil).Times(1)
294+
mockSubnetClient.EXPECT().CreateOrUpdate(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1)
295+
296+
err := d.updateSubnetServiceEndpoints(ctx)
297+
if !reflect.DeepEqual(err, nil) {
298+
t.Errorf("Unexpected error: %v", err)
299+
}
300+
},
301+
},
302+
{
303+
name: "[success] ServiceEndpoints is nil",
304+
testFunc: func(t *testing.T) {
305+
fakeSubnet := network.Subnet{
306+
SubnetPropertiesFormat: &network.SubnetPropertiesFormat{},
307+
}
308+
309+
mockSubnetClient.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(fakeSubnet, nil).Times(1)
310+
mockSubnetClient.EXPECT().CreateOrUpdate(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1)
311+
312+
err := d.updateSubnetServiceEndpoints(ctx)
313+
if !reflect.DeepEqual(err, nil) {
314+
t.Errorf("Unexpected error: %v", err)
315+
}
316+
},
317+
},
318+
{
319+
name: "[success] storageService does not exists",
320+
testFunc: func(t *testing.T) {
321+
fakeSubnet := network.Subnet{
322+
SubnetPropertiesFormat: &network.SubnetPropertiesFormat{
323+
ServiceEndpoints: &[]network.ServiceEndpointPropertiesFormat{},
324+
},
325+
}
326+
327+
mockSubnetClient.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(fakeSubnet, nil).Times(1)
328+
mockSubnetClient.EXPECT().CreateOrUpdate(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1)
329+
330+
err := d.updateSubnetServiceEndpoints(ctx)
331+
if !reflect.DeepEqual(err, nil) {
332+
t.Errorf("Unexpected error: %v", err)
333+
}
334+
},
335+
},
336+
{
337+
name: "[success] storageService already exists",
338+
testFunc: func(t *testing.T) {
339+
fakeSubnet := network.Subnet{
340+
SubnetPropertiesFormat: &network.SubnetPropertiesFormat{
341+
ServiceEndpoints: &[]network.ServiceEndpointPropertiesFormat{
342+
{
343+
Service: &storageService,
344+
},
345+
},
346+
},
347+
}
348+
349+
mockSubnetClient.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(fakeSubnet, nil).Times(1)
350+
351+
err := d.updateSubnetServiceEndpoints(ctx)
352+
if !reflect.DeepEqual(err, nil) {
353+
t.Errorf("Unexpected error: %v", err)
354+
}
355+
},
356+
},
357+
{
358+
name: "[fail] SubnetsClient is nil",
359+
testFunc: func(t *testing.T) {
360+
d.cloud.SubnetsClient = nil
361+
expectedErr := fmt.Errorf("SubnetsClient is nil")
362+
err := d.updateSubnetServiceEndpoints(ctx)
363+
if !reflect.DeepEqual(err, expectedErr) {
364+
t.Errorf("Unexpected error: %v", err)
365+
}
366+
},
367+
},
368+
}
369+
370+
for _, tc := range testCases {
371+
t.Run(tc.name, tc.testFunc)
372+
}
373+
}

‎pkg/blob/blob.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,8 @@ type Driver struct {
106106
// A map storing all volumes with ongoing operations so that additional operations
107107
// for that same volume (as defined by VolumeID) return an Aborted error
108108
volumeLocks *volumeLocks
109+
// only for nfs feature
110+
subnetLockMap *util.LockMap
109111
}
110112

111113
// NewDriver Creates a NewCSIDriver object. Assumes vendor version is equal to driver version &
@@ -116,6 +118,7 @@ func NewDriver(nodeID, blobfuseProxyEndpoint string, enableBlobfuseProxy bool, b
116118
driver.Version = driverVersion
117119
driver.NodeID = nodeID
118120
driver.volLockMap = util.NewLockMap()
121+
driver.subnetLockMap = util.NewLockMap()
119122
driver.volumeLocks = newVolumeLocks()
120123
driver.blobfuseProxyEndpoint = blobfuseProxyEndpoint
121124
driver.enableBlobfuseProxy = enableBlobfuseProxy

‎pkg/blob/blob_test.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ import (
3232
"k8s.io/client-go/kubernetes"
3333
"k8s.io/client-go/kubernetes/fake"
3434

35+
"sigs.k8s.io/blob-csi-driver/pkg/util"
3536
"sigs.k8s.io/cloud-provider-azure/pkg/azureclients/storageaccountclient/mockstorageaccountclient"
3637
azure "sigs.k8s.io/cloud-provider-azure/pkg/provider"
3738
"sigs.k8s.io/cloud-provider-azure/pkg/retry"
@@ -47,6 +48,7 @@ func NewFakeDriver() *Driver {
4748
driver := NewDriver(fakeNodeID, "", false, 5, false)
4849
driver.Name = fakeDriverName
4950
driver.Version = vendorVersion
51+
driver.subnetLockMap = util.NewLockMap()
5052
return driver
5153
}
5254

‎pkg/blob/controllerserver.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,9 @@ func (d *Driver) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest)
123123
vnetResourceID := d.getSubnetResourceID()
124124
klog.V(2).Infof("set vnetResourceID(%s) for NFS protocol", vnetResourceID)
125125
vnetResourceIDs = []string{vnetResourceID}
126+
if err := d.updateSubnetServiceEndpoints(ctx); err != nil {
127+
return nil, status.Errorf(codes.Internal, "update service endpoints failed with error: %v", err)
128+
}
126129
// NFS protocol does not need account key
127130
storeAccountKey = storeAccountKeyFalse
128131
}

‎test/e2e/dynamic_provisioning_test.go

Lines changed: 34 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ var _ = ginkgo.Describe("[blob-csi-e2e] Dynamic Provisioning", func() {
5656
})
5757

5858
testDriver = driver.InitBlobCSIDriver()
59-
ginkgo.It("should create a volume on demand", func() {
59+
ginkgo.It("should create a volume on demand with mount options", func() {
6060
pods := []testsuites.PodDetails{
6161
{
6262
Cmd: "echo 'hello world' > /mnt/test-1/data && grep 'hello world' /mnt/test-1/data",
@@ -277,6 +277,38 @@ var _ = ginkgo.Describe("[blob-csi-e2e] Dynamic Provisioning", func() {
277277
test.Run(cs, ns)
278278
})
279279

280+
ginkgo.It("should create a NFSv3 volume on demand with mount options [nfs]", func() {
281+
if isAzureStackCloud {
282+
ginkgo.Skip("test case is not available for Azure Stack")
283+
}
284+
pods := []testsuites.PodDetails{
285+
{
286+
Cmd: "echo 'hello world' > /mnt/test-1/data && grep 'hello world' /mnt/test-1/data",
287+
Volumes: []testsuites.VolumeDetails{
288+
{
289+
ClaimSize: "10Gi",
290+
MountOptions: []string{
291+
"nconnect=16",
292+
},
293+
VolumeMount: testsuites.VolumeMountDetails{
294+
NameGenerate: "test-volume-",
295+
MountPathGenerate: "/mnt/test-",
296+
},
297+
},
298+
},
299+
},
300+
}
301+
test := testsuites.DynamicallyProvisionedCmdVolumeTest{
302+
CSIDriver: testDriver,
303+
Pods: pods,
304+
StorageClassParameters: map[string]string{
305+
"skuName": "Premium_LRS",
306+
"protocol": "nfs",
307+
},
308+
}
309+
test.Run(cs, ns)
310+
})
311+
280312
ginkgo.It("should create a volume on demand (Bring Your Own Key)", func() {
281313
// get storage account secret name
282314
err := os.Chdir("../..")
@@ -325,7 +357,7 @@ var _ = ginkgo.Describe("[blob-csi-e2e] Dynamic Provisioning", func() {
325357
test.Run(cs, ns)
326358
})
327359

328-
ginkgo.It("should create a volume on demand and resize it [kubernetes.io/blob-csi] [blob.csi.azure.com]", func() {
360+
ginkgo.It("should create a volume on demand and resize it [blob.csi.azure.com]", func() {
329361
pods := []testsuites.PodDetails{
330362
{
331363
Cmd: "echo 'hello world' > /mnt/test-1/data && grep 'hello world' /mnt/test-1/data",

‎test/utils/blob_log.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ kubectl get pods -n default -o wide
2828
echo "======================================================================================"
2929

3030
echo "print out all $NS namespace pods status ..."
31-
kubectl get pods -n${NS}
31+
kubectl get pods -n${NS} -o wide
3232
echo "======================================================================================"
3333

3434
echo "print out csi-blob-controller logs ..."

0 commit comments

Comments
 (0)