Merge pull request #447 from andyzhangx/fsgrouppolicy

feat: support fsGroupPolicy for NFS

Author: andyzhangx

Makefile

@@ -33,6 +33,7 @@ E2E_HELM_OPTIONS ?= --set image.blob.pullPolicy=Always --set image.blob.reposito
 ifdef ENABLE_BLOBFUSE_PROXY
 override E2E_HELM_OPTIONS := $(E2E_HELM_OPTIONS) --set controller.logLevel=6 --set node.logLevel=6 --set node.enableBlobfuseProxy=true
 endif
+E2E_HELM_OPTIONS += ${EXTRA_HELM_OPTIONS}
 GINKGO_FLAGS = -ginkgo.v
 GO111MODULE = on
 GOPATH ?= $(shell go env GOPATH)
@@ -66,7 +67,7 @@ integration-test: blob
 
 .PHONY: e2e-test
 e2e-test:
-	if [ ! -z "$(EXTERNAL_E2E_TEST)" ]; then \
+	if [ ! -z "$(EXTERNAL_E2E_TEST_BLOBFUSE)" ] || [ ! -z "$(EXTERNAL_E2E_TEST_NFS)" ]; then \
 		bash ./test/external-e2e/run.sh;\
 	else \
 		go test -v -timeout=0 ./test/e2e ${GINKGO_FLAGS};\

charts/README.md

@@ -6,6 +6,11 @@ From `v0.7.0`, driver name changed from `blobfuse.csi.azure.com` to `blob.csi.az
 ## Prerequisites
  - [install Helm](https://helm.sh/docs/intro/quickstart/#install-helm)
 
+### Tips
+ - `--set controller.runOnMaster=true` could make csi-azuredisk-controller only run on master node
+ - `--set feature.enableFSGroupPolicy=true` could enable `fsGroupPolicy` on a k8s 1.20+ cluster
+ - `--set controller.replicas=1` could set replica of csi-azuredisk-controller as `1`
+
 ## install latest version
 ```console
 helm repo add blob-csi-driver https://raw.githubusercontent.com/kubernetes-sigs/blob-csi-driver/master/charts
@@ -44,6 +49,7 @@ The following table lists the configurable parameters of the latest Azure Blob S
 
 | Parameter                                             | Description                                           | Default                                                        |
 | ----------------------------------------------------- | ----------------------------------------------------- | -------------------------------------------------------------- |
+| `feature.enableFSGroupPolicy`                     | enable `fsGroupPolicy` on a k8s 1.20+ cluster           | `false`                      |
 | `image.blob.repository`                               | blob-csi-driver docker image                          | mcr.microsoft.com/k8s/csi/blob-csi                             |
 | `image.blob.tag`                                      | blob-csi-driver docker image tag                      | latest                                                         |
 | `image.blob.pullPolicy`                               | blob-csi-driver image pull policy                     | IfNotPresent                                                   |

charts/latest/blob-csi-driver-v1.3.0.tgz

@@ -6,6 +6,9 @@ metadata:
 spec:
   attachRequired: false
   podInfoOnMount: true
+  {{- if .Values.feature.enableFSGroupPolicy}}
+  fsGroupPolicy: File
+  {{- end}}
   volumeLifecycleModes:
     - Persistent
     - Ephemeral

charts/latest/blob-csi-driver/templates/csi-blob-driver.yaml

@@ -102,6 +102,9 @@ node:
   livenessProbe:
     healthPort: 29633
 
+feature:
+  enableFSGroupPolicy: false
+
 linux:
   kubelet: /var/lib/kubelet
   distro: debian

charts/latest/blob-csi-driver/values.yaml

@@ -28,7 +28,12 @@ setup_e2e_binaries() {
     curl -sL https://storage.googleapis.com/kubernetes-release/release/v1.21.0/kubernetes-test-linux-amd64.tar.gz --output e2e-tests.tar.gz
     tar -xvf e2e-tests.tar.gz && rm e2e-tests.tar.gz
 
-    # install blob csi driver
+    if [ ! -z ${EXTERNAL_E2E_TEST_NFS} ]; then
+        # enable fsGroupPolicy (only available from k8s 1.20)
+        export EXTRA_HELM_OPTIONS="--set feature.enableFSGroupPolicy=true"
+    fi
+
+    # install csi driver
     make e2e-bootstrap
     make create-metrics-svc
 }
@@ -44,16 +49,20 @@ trap print_logs EXIT
 
 mkdir -p /tmp/csi
 
-echo "begin to run blobfuse tests ...."
-cp deploy/example/storageclass-blobfuse.yaml /tmp/csi/storageclass.yaml
-ginkgo -p --progress --v -focus='External.Storage.*blob.csi.azure.com' \
-       -skip='\[Disruptive\]|\[Slow\]|allow exec of files on the volume|unmount after the subpath directory is deleted' kubernetes/test/bin/e2e.test  -- \
-       -storage.testdriver=$PROJECT_ROOT/test/external-e2e/testdriver.yaml \
-       --kubeconfig=$KUBECONFIG
-
-echo "begin to run NFSv3 tests ...."
-cp deploy/example/storageclass-blob-nfs.yaml /tmp/csi/storageclass.yaml
-ginkgo -p --progress --v -focus='External.Storage.*blob.csi.azure.com' \
-       -skip='\[Disruptive\]|\[Slow\]' kubernetes/test/bin/e2e.test  -- \
-       -storage.testdriver=$PROJECT_ROOT/test/external-e2e/testdriver.yaml \
-       --kubeconfig=$KUBECONFIG
+if [ ! -z ${EXTERNAL_E2E_TEST_BLOBFUSE} ]; then
+    echo "begin to run blobfuse tests ...."
+    cp deploy/example/storageclass-blobfuse.yaml /tmp/csi/storageclass.yaml
+    ginkgo -p --progress --v -focus='External.Storage.*blob.csi.azure.com' \
+        -skip='\[Disruptive\]|\[Slow\]|allow exec of files on the volume|unmount after the subpath directory is deleted' kubernetes/test/bin/e2e.test  -- \
+        -storage.testdriver=$PROJECT_ROOT/test/external-e2e/testdriver-blobfuse.yaml \
+        --kubeconfig=$KUBECONFIG
+fi
+
+if [ ! -z ${EXTERNAL_E2E_TEST_NFS} ]; then
+    echo "begin to run NFSv3 tests ...."
+    cp deploy/example/storageclass-blob-nfs.yaml /tmp/csi/storageclass.yaml
+    ginkgo -p --progress --v -focus='External.Storage.*blob.csi.azure.com' \
+        -skip='\[Disruptive\]|\[Slow\]|pod created with an initial fsgroup, volume contents ownership changed in first pod, new pod with same fsgroup skips ownership changes to the volume contents' kubernetes/test/bin/e2e.test  -- \
+        -storage.testdriver=$PROJECT_ROOT/test/external-e2e/testdriver-nfs.yaml \
+        --kubeconfig=$KUBECONFIG
+fi

test/external-e2e/run.sh

@@ -0,0 +1,19 @@
+# Manifest for Kubernetes external tests.
+# See https://github.com/kubernetes/kubernetes/tree/master/test/e2e/storage/external
+
+ShortName: blobfuse
+StorageClass:
+  FromFile: /tmp/csi/storageclass.yaml
+DriverInfo:
+  Name: blob.csi.azure.com
+  Capabilities:
+    persistence: true
+    exec: true
+    multipods: true
+    RWX: true
+    fsGroup: true
+    topology: false
+    controllerExpansion: true
+    nodeExpansion: true
+    volumeLimits: false
+    snapshotDataSource: false