Skip to content

Commit 33120c3

Browse files
andyzhangxandyzhangx
authored
Merge pull request #434 from andyzhangx/inline-volume
feat: support inline volume
2 parents c705089 + b4a4b53 commit 33120c3

File tree

12 files changed

+263
-46
lines changed

12 files changed

+263
-46
lines changed

‎charts/latest/blob-csi-driver-v1.2.0.tgz

24 Bytes
Binary file not shown.

‎charts/latest/blob-csi-driver/templates/csi-blob-driver.yaml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,3 +6,6 @@ metadata:
66
spec:
77
attachRequired: false
88
podInfoOnMount: true
9+
volumeLifecycleModes:
10+
- Persistent
11+
- Ephemeral

‎deploy/csi-blob-driver.yaml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,3 +6,6 @@ metadata:
66
spec:
77
attachRequired: false
88
podInfoOnMount: true
9+
volumeLifecycleModes:
10+
- Persistent
11+
- Ephemeral

‎deploy/example/nginx-blobfuse-inline-volume.yaml

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
---
2+
kind: Pod
3+
apiVersion: v1
4+
metadata:
5+
name: nginx-blobfuse-inline-volume
6+
spec:
7+
nodeSelector:
8+
"kubernetes.io/os": linux
9+
containers:
10+
- image: mcr.microsoft.com/oss/nginx/nginx:1.19.5
11+
name: nginx-blobfuse
12+
command:
13+
- "/bin/bash"
14+
- "-c"
15+
- set -euo pipefail; while true; do echo $(date) >> /mnt/blobfuse/outfile; sleep 1; done
16+
volumeMounts:
17+
- name: persistent-storage
18+
mountPath: "/mnt/blobfuse"
19+
volumes:
20+
- name: persistent-storage
21+
csi:
22+
driver: blob.csi.azure.com
23+
volumeAttributes:
24+
containerName: EXISTING_CONTAINER_NAME
25+
secretName: azure-secret
26+
secretNamespace: default # optional, if it's empty, use pod.Namespace by default

‎pkg/blob/blob.go

Lines changed: 19 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ const (
5757
skuNameField = "skuname"
5858
resourceGroupField = "resourcegroup"
5959
locationField = "location"
60+
secretNameField = "secretname"
6061
secretNamespaceField = "secretnamespace"
6162
containerNameField = "containername"
6263
storeAccountKeyField = "storeaccountkey"
@@ -236,6 +237,8 @@ func (d *Driver) GetAuthEnv(ctx context.Context, volumeID, protocol string, attr
236237
var (
237238
accountKey string
238239
accountSasToken string
240+
secretName string
241+
secretNamespace string
239242
keyVaultURL string
240243
keyVaultSecretName string
241244
keyVaultSecretVersion string
@@ -256,6 +259,10 @@ func (d *Driver) GetAuthEnv(ctx context.Context, volumeID, protocol string, attr
256259
accountName = v
257260
case storageAccountNameField: // for compatibility
258261
accountName = v
262+
case secretNameField:
263+
secretName = v
264+
case secretNamespaceField:
265+
secretNamespace = v
259266
case "azurestorageauthtype":
260267
authEnv = append(authEnv, "AZURE_STORAGE_AUTH_TYPE="+v)
261268
case "azurestorageidentityclientid":
@@ -297,7 +304,8 @@ func (d *Driver) GetAuthEnv(ctx context.Context, volumeID, protocol string, attr
297304
} else {
298305
if len(secrets) == 0 {
299306
// read from k8s secret first
300-
accountKey, err = d.GetStorageAccesskeyFromSecret(accountName, attrib[secretNamespaceField])
307+
var name string
308+
name, accountKey, err = d.GetStorageAccountFromSecret(secretName, secretNamespace)
301309
if err != nil {
302310
klog.V(2).Infof("could not get account(%s) key from secret, error: %v, use cluster identity to get account key instead", accountName, err)
303311
if rgName == "" {
@@ -308,6 +316,9 @@ func (d *Driver) GetAuthEnv(ctx context.Context, volumeID, protocol string, attr
308316
return accountName, containerName, authEnv, fmt.Errorf("no key for storage account(%s) under resource group(%s), err %v", accountName, rgName, err)
309317
}
310318
}
319+
if name != "" {
320+
accountName = name
321+
}
311322
} else {
312323
for k, v := range secrets {
313324
switch strings.ToLower(k) {
@@ -516,30 +527,27 @@ func (d *Driver) GetStorageAccesskey(accountOptions *azure.AccountOptions, secre
516527
}
517528

518529
// read from k8s secret first
519-
accountKey, err := d.GetStorageAccesskeyFromSecret(accountOptions.Name, secretNamespace)
530+
_, accountKey, err := d.GetStorageAccountFromSecret(accountOptions.Name, secretNamespace)
520531
if err != nil {
521532
klog.V(2).Infof("could not get account(%s) key from secret, error: %v, use cluster identity to get account key instead", accountOptions.Name, err)
522533
accountKey, err = d.cloud.GetStorageAccesskey(accountOptions.Name, accountOptions.ResourceGroup)
523534
}
524535
return accountOptions.Name, accountKey, err
525536
}
526537

527-
// GetStorageAccesskeyFromSecret get storage account key from k8s secret
528-
func (d *Driver) GetStorageAccesskeyFromSecret(accountName, secretNamespace string) (string, error) {
538+
// GetStorageAccountFromSecret get storage account key from k8s secret
539+
// return <accountName, accountKey, error>
540+
func (d *Driver) GetStorageAccountFromSecret(secretName, secretNamespace string) (string, string, error) {
529541
if d.cloud.KubeClient == nil {
530-
return "", fmt.Errorf("could not get account(%s) key from secret: KubeClient is nil", accountName)
542+
return "", "", fmt.Errorf("could not get account key from secret(%s): KubeClient is nil", secretName)
531543
}
532544

533-
secretName := fmt.Sprintf(secretNameTemplate, accountName)
534-
if secretNamespace == "" {
535-
secretNamespace = defaultSecretNamespace
536-
}
537545
secret, err := d.cloud.KubeClient.CoreV1().Secrets(secretNamespace).Get(context.TODO(), secretName, metav1.GetOptions{})
538546
if err != nil {
539-
return "", fmt.Errorf("could not get secret(%v): %v", secretName, err)
547+
return "", "", fmt.Errorf("could not get secret(%v): %v", secretName, err)
540548
}
541549

542-
return string(secret.Data[defaultSecretAccountKey][:]), nil
550+
return string(secret.Data[defaultSecretAccountName][:]), string(secret.Data[defaultSecretAccountKey][:]), nil
543551
}
544552

545553
// getSubnetResourceID get default subnet resource ID from cloud provider config

‎pkg/blob/nodeserver.go

Lines changed: 28 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -54,24 +54,46 @@ func NewMountClient(cc *grpc.ClientConn) *MountClient {
5454

5555
// NodePublishVolume mount the volume from staging to target path
5656
func (d *Driver) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) {
57-
if req.GetVolumeCapability() == nil {
57+
volCap := req.GetVolumeCapability()
58+
if volCap == nil {
5859
return nil, status.Error(codes.InvalidArgument, "Volume capability missing in request")
5960
}
6061
volumeID := req.GetVolumeId()
6162
if len(req.GetVolumeId()) == 0 {
6263
return nil, status.Error(codes.InvalidArgument, "Volume ID missing in request")
6364
}
6465

65-
source := req.GetStagingTargetPath()
66-
if len(source) == 0 {
67-
return nil, status.Error(codes.InvalidArgument, "Staging target not provided")
68-
}
69-
7066
target := req.GetTargetPath()
7167
if len(target) == 0 {
7268
return nil, status.Error(codes.InvalidArgument, "Target path not provided")
7369
}
7470

71+
context := req.GetVolumeContext()
72+
if context != nil && context["csi.storage.k8s.io/ephemeral"] == "true" {
73+
// if secretNamespace is not set, set same namespace as pod
74+
secretNamespace := context["csi.storage.k8s.io/pod.namespace"]
75+
for k, v := range context {
76+
switch strings.ToLower(k) {
77+
case secretNamespaceField:
78+
secretNamespace = v
79+
}
80+
}
81+
context[secretNamespaceField] = secretNamespace
82+
klog.V(2).Infof("NodePublishVolume: ephemeral volume(%s) mount on %s, VolumeContext: %v", volumeID, target, context)
83+
_, err := d.NodeStageVolume(ctx, &csi.NodeStageVolumeRequest{
84+
StagingTargetPath: target,
85+
VolumeContext: context,
86+
VolumeCapability: volCap,
87+
VolumeId: volumeID,
88+
})
89+
return &csi.NodePublishVolumeResponse{}, err
90+
}
91+
92+
source := req.GetStagingTargetPath()
93+
if len(source) == 0 {
94+
return nil, status.Error(codes.InvalidArgument, "Staging target not provided")
95+
}
96+
7597
mountOptions := []string{"bind"}
7698
if req.GetReadonly() {
7799
mountOptions = append(mountOptions, "ro")

‎pkg/blob/nodeserver_test.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -172,7 +172,8 @@ func TestNodePublishVolume(t *testing.T) {
172172
{
173173
desc: "Stage path missing",
174174
req: csi.NodePublishVolumeRequest{VolumeCapability: &csi.VolumeCapability{AccessMode: &volumeCap},
175-
VolumeId: "vol_1"},
175+
VolumeId: "vol_1",
176+
TargetPath: sourceTest},
176177
expectedErr: status.Error(codes.InvalidArgument, "Staging target not provided"),
177178
},
178179
{

‎test/e2e/dynamic_provisioning_test.go

Lines changed: 88 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ limitations under the License.
1717
package e2e
1818

1919
import (
20+
"context"
2021
"fmt"
2122
"log"
2223
"os"
@@ -277,18 +278,37 @@ var _ = ginkgo.Describe("[blob-csi-e2e] Dynamic Provisioning", func() {
277278
test.Run(cs, ns)
278279
})
279280

280-
ginkgo.It("should create a NFSv3 volume on demand with mount options [nfs]", func() {
281-
if isAzureStackCloud {
282-
ginkgo.Skip("test case is not available for Azure Stack")
283-
}
281+
ginkgo.It("should create a volume on demand (Bring Your Own Key)", func() {
282+
// get storage account secret name
283+
err := os.Chdir("../..")
284+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
285+
defer func() {
286+
err := os.Chdir("test/e2e")
287+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
288+
}()
289+
290+
getSecretNameScript := "test/utils/get_storage_account_secret_name.sh"
291+
log.Printf("run script: %s\n", getSecretNameScript)
292+
293+
cmd := exec.Command("bash", getSecretNameScript)
294+
output, err := cmd.CombinedOutput()
295+
log.Printf("got output: %v, error: %v\n", string(output), err)
296+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
297+
298+
secretName := strings.TrimSuffix(string(output), "\n")
299+
log.Printf("got storage account secret name: %v\n", secretName)
300+
bringKeyStorageClassParameters["csi.storage.k8s.io/provisioner-secret-name"] = secretName
301+
bringKeyStorageClassParameters["csi.storage.k8s.io/node-stage-secret-name"] = secretName
302+
284303
pods := []testsuites.PodDetails{
285304
{
286305
Cmd: "echo 'hello world' > /mnt/test-1/data && grep 'hello world' /mnt/test-1/data",
287306
Volumes: []testsuites.VolumeDetails{
288307
{
289308
ClaimSize: "10Gi",
290309
MountOptions: []string{
291-
"nconnect=16",
310+
"-o allow_other",
311+
"--file-cache-timeout-in-seconds=120",
292312
},
293313
VolumeMount: testsuites.VolumeMountDetails{
294314
NameGenerate: "test-volume-",
@@ -299,17 +319,37 @@ var _ = ginkgo.Describe("[blob-csi-e2e] Dynamic Provisioning", func() {
299319
},
300320
}
301321
test := testsuites.DynamicallyProvisionedCmdVolumeTest{
302-
CSIDriver: testDriver,
303-
Pods: pods,
304-
StorageClassParameters: map[string]string{
305-
"skuName": "Premium_LRS",
306-
"protocol": "nfs",
322+
CSIDriver: testDriver,
323+
Pods: pods,
324+
StorageClassParameters: bringKeyStorageClassParameters,
325+
}
326+
test.Run(cs, ns)
327+
})
328+
329+
ginkgo.It("should create a volume on demand and resize it [blob.csi.azure.com]", func() {
330+
pods := []testsuites.PodDetails{
331+
{
332+
Cmd: "echo 'hello world' > /mnt/test-1/data && grep 'hello world' /mnt/test-1/data",
333+
Volumes: []testsuites.VolumeDetails{
334+
{
335+
ClaimSize: "10Gi",
336+
VolumeMount: testsuites.VolumeMountDetails{
337+
NameGenerate: "test-volume-",
338+
MountPathGenerate: "/mnt/test-",
339+
},
340+
},
341+
},
307342
},
308343
}
344+
test := testsuites.DynamicallyProvisionedResizeVolumeTest{
345+
CSIDriver: testDriver,
346+
Pods: pods,
347+
StorageClassParameters: map[string]string{"skuName": "Standard_LRS"},
348+
}
309349
test.Run(cs, ns)
310350
})
311351

312-
ginkgo.It("should create a volume on demand (Bring Your Own Key)", func() {
352+
ginkgo.It("should create an CSI inline volume [blob.csi.azure.com]", func() {
313353
// get storage account secret name
314354
err := os.Chdir("../..")
315355
gomega.Expect(err).NotTo(gomega.HaveOccurred())
@@ -328,15 +368,27 @@ var _ = ginkgo.Describe("[blob-csi-e2e] Dynamic Provisioning", func() {
328368

329369
secretName := strings.TrimSuffix(string(output), "\n")
330370
log.Printf("got storage account secret name: %v\n", secretName)
331-
bringKeyStorageClassParameters["csi.storage.k8s.io/provisioner-secret-name"] = secretName
332-
bringKeyStorageClassParameters["csi.storage.k8s.io/node-stage-secret-name"] = secretName
371+
segments := strings.Split(secretName, "-")
372+
if len(segments) != 5 {
373+
ginkgo.Fail(fmt.Sprintf("%s have %d elements, expected: %d ", secretName, len(segments), 5))
374+
}
375+
accountName := segments[3]
376+
377+
containerName := "csi-inline-blobfuse-volume"
378+
req := makeCreateVolumeReq(containerName)
379+
req.Parameters["storageAccount"] = accountName
380+
resp, err := blobDriver.CreateVolume(context.Background(), req)
381+
if err != nil {
382+
ginkgo.Fail(fmt.Sprintf("create volume error: %v", err))
383+
}
384+
volumeID := resp.Volume.VolumeId
385+
ginkgo.By(fmt.Sprintf("Successfully provisioned Blobfuse volume: %q\n", volumeID))
333386

334387
pods := []testsuites.PodDetails{
335388
{
336-
Cmd: "echo 'hello world' > /mnt/test-1/data && grep 'hello world' /mnt/test-1/data",
337389
Volumes: []testsuites.VolumeDetails{
338390
{
339-
ClaimSize: "10Gi",
391+
ClaimSize: "100Gi",
340392
MountOptions: []string{
341393
"-o allow_other",
342394
"--file-cache-timeout-in-seconds=120",
@@ -349,21 +401,30 @@ var _ = ginkgo.Describe("[blob-csi-e2e] Dynamic Provisioning", func() {
349401
},
350402
},
351403
}
352-
test := testsuites.DynamicallyProvisionedCmdVolumeTest{
353-
CSIDriver: testDriver,
354-
Pods: pods,
355-
StorageClassParameters: bringKeyStorageClassParameters,
404+
405+
test := testsuites.DynamicallyProvisionedInlineVolumeTest{
406+
CSIDriver: testDriver,
407+
Pods: pods,
408+
SecretName: secretName,
409+
ContainerName: containerName,
410+
ReadOnly: false,
356411
}
357412
test.Run(cs, ns)
358413
})
359414

360-
ginkgo.It("should create a volume on demand and resize it [blob.csi.azure.com]", func() {
415+
ginkgo.It("should create a NFSv3 volume on demand with mount options [nfs]", func() {
416+
if isAzureStackCloud {
417+
ginkgo.Skip("test case is not available for Azure Stack")
418+
}
361419
pods := []testsuites.PodDetails{
362420
{
363421
Cmd: "echo 'hello world' > /mnt/test-1/data && grep 'hello world' /mnt/test-1/data",
364422
Volumes: []testsuites.VolumeDetails{
365423
{
366424
ClaimSize: "10Gi",
425+
MountOptions: []string{
426+
"nconnect=16",
427+
},
367428
VolumeMount: testsuites.VolumeMountDetails{
368429
NameGenerate: "test-volume-",
369430
MountPathGenerate: "/mnt/test-",
@@ -372,10 +433,13 @@ var _ = ginkgo.Describe("[blob-csi-e2e] Dynamic Provisioning", func() {
372433
},
373434
},
374435
}
375-
test := testsuites.DynamicallyProvisionedResizeVolumeTest{
376-
CSIDriver: testDriver,
377-
Pods: pods,
378-
StorageClassParameters: map[string]string{"skuName": "Standard_LRS"},
436+
test := testsuites.DynamicallyProvisionedCmdVolumeTest{
437+
CSIDriver: testDriver,
438+
Pods: pods,
439+
StorageClassParameters: map[string]string{
440+
"skuName": "Premium_LRS",
441+
"protocol": "nfs",
442+
},
379443
}
380444
test.Run(cs, ns)
381445
})

0 commit comments

Comments
 (0)