feat: Implement support for expanding volume

Saurav Tiwary · Saurav Tiwary · commit 38d6114486e3 · 2020-12-13T16:49:22.000+05:30
diff --git a/charts/latest/blob-csi-driver/templates/csi-blob-controller.yaml b/charts/latest/blob-csi-driver/templates/csi-blob-controller.yaml
@@ -119,6 +119,26 @@ spec:
             requests:
               cpu: 10m
               memory: 20Mi
+        - name: csi-resizer
+          image: "{{ .Values.image.csiResizer.repository }}:{{ .Values.image.csiResizer.tag }}"
+          args:
+            - "-csi-address=$(ADDRESS)"
+            - "-v=5"
+            - "-leader-election"
+          env:
+            - name: ADDRESS
+              value: /csi/csi.sock
+          imagePullPolicy: {{ .Values.image.csiResizer.pullPolicy }}
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+          resources:
+            limits:
+              cpu: 100m
+              memory: 100Mi
+            requests:
+              cpu: 10m
+              memory: 20Mi
       volumes:
         - name: socket-dir
           emptyDir: {}
diff --git a/charts/latest/blob-csi-driver/templates/rbac-csi-blob-controller.yaml b/charts/latest/blob-csi-driver/templates/rbac-csi-blob-controller.yaml
@@ -46,6 +46,46 @@ roleRef:
 
 ---
 
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: blob-external-resizer-role
+  namespace: {{ .Release.Namespace }}
+{{ include "blob.labels" . | indent 2 }}
+rules:
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch", "update", "patch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims/status"]
+    verbs: ["update", "patch"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["list", "watch", "create", "update", "patch"]
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["leases"]
+    verbs: ["get", "list", "watch", "create", "update", "patch"]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: blob-csi-resizer-role
+  namespace: {{ .Release.Namespace }}
+{{ include "blob.labels" . | indent 2 }}
+subjects:
+  - kind: ServiceAccount
+    name: csi-blob-controller-sa
+    namespace: {{ .Release.Namespace }}
+roleRef:
+  kind: ClusterRole
+  name: blob-external-resizer-role
+  apiGroup: rbac.authorization.k8s.io
+
+---
+
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
diff --git a/charts/latest/blob-csi-driver/values.yaml b/charts/latest/blob-csi-driver/values.yaml
@@ -15,6 +15,10 @@ image:
     repository: mcr.microsoft.com/oss/kubernetes-csi/csi-node-driver-registrar
     tag: v2.0.1
     pullPolicy: IfNotPresent
+  csiResizer:
+    repository: mcr.microsoft.com/oss/kubernetes-csi/csi-resizer
+    tag: v0.5.0
+    pullPolicy: IfNotPresent
 
 serviceAccount:
   create: true
diff --git a/deploy/csi-blob-controller.yaml b/deploy/csi-blob-controller.yaml
@@ -113,6 +113,25 @@ spec:
             requests:
               cpu: 10m
               memory: 20Mi
+        - name: csi-resizer
+          image: mcr.microsoft.com/oss/kubernetes-csi/csi-resizer:v0.5.0
+          args:
+            - "-csi-address=$(ADDRESS)"
+            - "-v=5"
+            - "-leader-election"
+          env:
+            - name: ADDRESS
+              value: /csi/csi.sock
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+          resources:
+            limits:
+              cpu: 100m
+              memory: 100Mi
+            requests:
+              cpu: 10m
+              memory: 20Mi
       volumes:
         - name: socket-dir
           emptyDir: {}
diff --git a/pkg/blob/blob.go b/pkg/blob/blob.go
@@ -83,6 +83,9 @@ const (
 	shareNotFound         = "The specified share does not exist"
 	shareBeingDeleted     = "The specified share is being deleted"
 	clientThrottled       = "client throttled"
+
+	// containerMaxSize is the max size of the blob container. See https://docs.microsoft.com/en-us/azure/storage/blobs/scalability-targets#scale-targets-for-blob-storage
+	containerMaxSize = 100 * util.TiB
 )
 
 var (
@@ -141,6 +144,7 @@ func (d *Driver) Run(endpoint, kubeconfig string, testBool bool) {
 			csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME,
 			//csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT,
 			//csi.ControllerServiceCapability_RPC_LIST_SNAPSHOTS,
+			csi.ControllerServiceCapability_RPC_EXPAND_VOLUME,
 		})
 	d.AddVolumeCapabilityAccessModes([]csi.VolumeCapability_AccessMode_Mode{
 		csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER,
diff --git a/pkg/blob/controllerserver.go b/pkg/blob/controllerserver.go
@@ -360,5 +360,26 @@ func (d *Driver) ListSnapshots(ctx context.Context, req *csi.ListSnapshotsReques
 
 // ControllerExpandVolume controller expand volume
 func (d *Driver) ControllerExpandVolume(ctx context.Context, req *csi.ControllerExpandVolumeRequest) (*csi.ControllerExpandVolumeResponse, error) {
-	return nil, status.Error(codes.Unimplemented, "ControllerExpandVolume is not yet implemented")
+	if len(req.GetVolumeId()) == 0 {
+		return nil, status.Error(codes.InvalidArgument, "Volume ID missing in request")
+	}
+
+	if req.GetCapacityRange() == nil {
+		return nil, status.Error(codes.InvalidArgument, "Capacity Range missing in request")
+	}
+
+	if err := d.ValidateControllerServiceRequest(csi.ControllerServiceCapability_RPC_EXPAND_VOLUME); err != nil {
+		return nil, fmt.Errorf("invalid expand volume req: %v", req)
+	}
+
+	volSizeBytes := int64(req.GetCapacityRange().GetRequiredBytes())
+	requestGiB := int64(util.RoundUpGiB(volSizeBytes))
+
+	if volSizeBytes > containerMaxSize {
+		return nil, status.Errorf(codes.OutOfRange, "required bytes (%d) exceeds the maximum supported bytes (%d)", volSizeBytes, containerMaxSize)
+	}
+
+	klog.V(2).Infof("ControllerExpandVolume(%s) successfully, currentQuota: %d Gi", req.VolumeId, requestGiB)
+
+	return &csi.ControllerExpandVolumeResponse{CapacityBytes: req.GetCapacityRange().GetRequiredBytes()}, nil
 }
diff --git a/pkg/blob/controllerserver_test.go b/pkg/blob/controllerserver_test.go
@@ -551,11 +551,54 @@ func TestListSnapshots(t *testing.T) {
 }
 
 func TestControllerExpandVolume(t *testing.T) {
-	d := NewFakeDriver()
-	req := csi.ControllerExpandVolumeRequest{}
-	resp, err := d.ControllerExpandVolume(context.Background(), &req)
-	assert.Nil(t, resp)
-	if !reflect.DeepEqual(err, status.Error(codes.Unimplemented, "ControllerExpandVolume is not yet implemented")) {
-		t.Errorf("Unexpected error: %v", err)
+	testCases := []struct {
+		name     string
+		testFunc func(t *testing.T)
+	}{
+		{
+			name: "volume ID missing",
+			testFunc: func(t *testing.T) {
+				d := NewFakeDriver()
+				req := &csi.ControllerExpandVolumeRequest{}
+				_, err := d.ControllerExpandVolume(context.Background(), req)
+				expectedErr := status.Error(codes.InvalidArgument, "Volume ID missing in request")
+				if !reflect.DeepEqual(err, expectedErr) {
+					t.Errorf("actualErr: (%v), expectedErr: (%v)", err, expectedErr)
+				}
+			},
+		},
+		{
+			name: "Capacity Range missing",
+			testFunc: func(t *testing.T) {
+				d := NewFakeDriver()
+				req := &csi.ControllerExpandVolumeRequest{
+					VolumeId: "unit-test",
+				}
+				_, err := d.ControllerExpandVolume(context.Background(), req)
+				expectedErr := status.Error(codes.InvalidArgument, "Capacity Range missing in request")
+				if !reflect.DeepEqual(err, expectedErr) {
+					t.Errorf("actualErr: (%v), expectedErr: (%v)", err, expectedErr)
+				}
+			},
+		},
+		{
+			name: "invalid expand volume req",
+			testFunc: func(t *testing.T) {
+				d := NewFakeDriver()
+				req := &csi.ControllerExpandVolumeRequest{
+					VolumeId:      "unit-test",
+					CapacityRange: &csi.CapacityRange{},
+				}
+				_, err := d.ControllerExpandVolume(context.Background(), req)
+				expectedErr := fmt.Errorf("invalid expand volume req: volume_id:\"unit-test\" capacity_range:<> ")
+				if !reflect.DeepEqual(err, expectedErr) {
+					t.Errorf("actualErr: (%v), expectedErr: (%v)", err, expectedErr)
+				}
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, tc.testFunc)
 	}
 }
diff --git a/pkg/util/util.go b/pkg/util/util.go
@@ -23,6 +23,7 @@ import (
 
 const (
 	GiB = 1024 * 1024 * 1024
+	TiB = 1024 * GiB
 )
 
 // RoundUpBytes rounds up the volume size in bytes upto multiplications of GiB
diff --git a/test/e2e/driver/driver.go b/test/e2e/driver/driver.go
@@ -58,15 +58,17 @@ func getStorageClass(
 		defaultBindingMode := storagev1.VolumeBindingImmediate
 		bindingMode = &defaultBindingMode
 	}
+	allowVolumeExpansion := true
 	return &storagev1.StorageClass{
 		ObjectMeta: metav1.ObjectMeta{
 			GenerateName: generateName,
 		},
-		Provisioner:       provisioner,
-		Parameters:        parameters,
-		MountOptions:      mountOptions,
-		ReclaimPolicy:     reclaimPolicy,
-		VolumeBindingMode: bindingMode,
-		AllowedTopologies: allowedTopologies,
+		Provisioner:          provisioner,
+		Parameters:           parameters,
+		MountOptions:         mountOptions,
+		ReclaimPolicy:        reclaimPolicy,
+		VolumeBindingMode:    bindingMode,
+		AllowedTopologies:    allowedTopologies,
+		AllowVolumeExpansion: &allowVolumeExpansion,
 	}
 }
diff --git a/test/e2e/dynamic_provisioning_test.go b/test/e2e/dynamic_provisioning_test.go
@@ -313,4 +313,27 @@ var _ = ginkgo.Describe("[blob-csi-e2e] Dynamic Provisioning", func() {
 		}
 		test.Run(cs, ns)
 	})
+
+	ginkgo.It("should create a volume on demand and resize it [kubernetes.io/blob-csi] [blob.csi.azure.com]", func() {
+		pods := []testsuites.PodDetails{
+			{
+				Cmd: "echo 'hello world' > /mnt/test-1/data && grep 'hello world' /mnt/test-1/data",
+				Volumes: []testsuites.VolumeDetails{
+					{
+						ClaimSize: "10Gi",
+						VolumeMount: testsuites.VolumeMountDetails{
+							NameGenerate:      "test-volume-",
+							MountPathGenerate: "/mnt/test-",
+						},
+					},
+				},
+			},
+		}
+		test := testsuites.DynamicallyProvisionedResizeVolumeTest{
+			CSIDriver:              testDriver,
+			Pods:                   pods,
+			StorageClassParameters: map[string]string{"skuName": "Standard_LRS"},
+		}
+		test.Run(cs, ns)
+	})
 })
diff --git a/test/e2e/testsuites/dynamically_provisioned_resize_volume_tester.go b/test/e2e/testsuites/dynamically_provisioned_resize_volume_tester.go
@@ -0,0 +1,100 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package testsuites
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/onsi/ginkgo"
+	"sigs.k8s.io/blob-csi-driver/pkg/util"
+	"sigs.k8s.io/blob-csi-driver/test/e2e/driver"
+
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/kubernetes/test/e2e/framework"
+)
+
+// DynamicallyProvisionedResizeVolumeTest will provision required StorageClass(es), PVC(s) and Pod(s)
+// Waiting for the PV provisioner to resize the PV
+// Testing if the PV is resized successfully.
+type DynamicallyProvisionedResizeVolumeTest struct {
+	CSIDriver              driver.DynamicPVTestDriver
+	Pods                   []PodDetails
+	StorageClassParameters map[string]string
+}
+
+func (t *DynamicallyProvisionedResizeVolumeTest) Run(client clientset.Interface, namespace *v1.Namespace) {
+	for _, pod := range t.Pods {
+		tpod, cleanup := pod.SetupWithDynamicVolumes(client, namespace, t.CSIDriver, t.StorageClassParameters)
+		// defer must be called here for resources not get removed before using them
+		for i := range cleanup {
+			defer cleanup[i]()
+		}
+
+		ginkgo.By("deploying the pod")
+		tpod.Create()
+		defer tpod.Cleanup()
+		ginkgo.By("checking that the pods command exits with no error")
+		tpod.WaitForSuccess()
+
+		pvcName := tpod.pod.Spec.Volumes[0].VolumeSource.PersistentVolumeClaim.ClaimName
+		pvc, err := client.CoreV1().PersistentVolumeClaims(namespace.Name).Get(context.TODO(), pvcName, metav1.GetOptions{})
+		if err != nil {
+			framework.ExpectNoError(err, fmt.Sprintf("fail to get original pvc(%s): %v", pvcName, err))
+		}
+
+		originalSize := pvc.Spec.Resources.Requests["storage"]
+		delta := resource.Quantity{}
+		delta.Set(util.GiBToBytes(1))
+		originalSize.Add(delta)
+		pvc.Spec.Resources.Requests["storage"] = originalSize
+
+		ginkgo.By("resizing the pvc")
+		updatedPvc, err := client.CoreV1().PersistentVolumeClaims(namespace.Name).Update(context.TODO(), pvc, metav1.UpdateOptions{})
+		if err != nil {
+			framework.ExpectNoError(err, fmt.Sprintf("fail to resize pvc(%s): %v", pvcName, err))
+		}
+		updatedSize := updatedPvc.Spec.Resources.Requests["storage"]
+
+		ginkgo.By("sleep 30s waiting for resize complete")
+		time.Sleep(30 * time.Second)
+
+		ginkgo.By("checking the resizing result")
+		newPvc, err := client.CoreV1().PersistentVolumeClaims(namespace.Name).Get(context.TODO(), tpod.pod.Spec.Volumes[0].VolumeSource.PersistentVolumeClaim.ClaimName, metav1.GetOptions{})
+		if err != nil {
+			framework.ExpectNoError(err, fmt.Sprintf("fail to get new pvc(%s): %v", pvcName, err))
+		}
+		newSize := newPvc.Spec.Resources.Requests["storage"]
+		if !newSize.Equal(updatedSize) {
+			framework.Failf("newSize(%+v) is not equal to updatedSize(%+v)", newSize, updatedSize)
+		}
+
+		ginkgo.By("checking the resizing PV result")
+		newPv, _ := client.CoreV1().PersistentVolumes().Get(context.Background(), updatedPvc.Spec.VolumeName, metav1.GetOptions{})
+		newPvSize := newPv.Spec.Capacity["storage"]
+		newPvSizeStr := newPvSize.String() + "Gi"
+
+		if !strings.Contains(newPvSizeStr, newSize.String()) {
+			framework.Failf("newPVCSize(%+v) is not equal to newPVSize(%+v)", newSize.String(), newPvSizeStr)
+		}
+	}
+}
diff --git a/test/integration/run-test.sh b/test/integration/run-test.sh
@@ -24,6 +24,7 @@ function cleanup {
 
 readonly volname="citest-$(date +%s)"
 readonly volsize="2147483648"
+readonly expanded_vol_size="2147483650"
 readonly endpoint="$1"
 readonly staging_target_path="$2"
 readonly target_path="$3"
@@ -70,6 +71,11 @@ if [[ "$cloud" != "AzureChinaCloud" ]]; then
   sleep 2
 fi
 
+# TODO: Activate test once issue is resolved - https://github.com/rexray/gocsi/issues/141
+# echo "Expand volume test:"
+# csc controller expand-volume --endpoint "$endpoint" --req-bytes "$expanded_vol_size" "$volumeid"
+# sleep 15
+
 echo "Delete volume test:"
 csc controller del --endpoint "$endpoint" "$volumeid"
 sleep 15

Original file line number	Diff line number	Diff line change
`@@ -23,6 +23,7 @@ import (`
`23`	`23`
`24`	`24`	`const (`
`25`	`25`	`GiB = 1024 * 1024 * 1024`
	`26`	`+ TiB = 1024 * GiB`
`26`	`27`	`)`
`27`	`28`
`28`	`29`	`// RoundUpBytes rounds up the volume size in bytes upto multiplications of GiB`