feat: bring own serviceAccount in helm install

andyzhangx · andyzhangx · commit 7f69a52322c0 · 2021-07-10T05:52:08.000Z
fix test

fix test failure

fix external test

fix test
diff --git a/Makefile b/Makefile
@@ -30,6 +30,7 @@ IMAGE_TAG_LATEST = $(REGISTRY)/$(IMAGE_NAME):latest
 BUILD_DATE ?= $(shell date -u +"%Y-%m-%dT%H:%M:%SZ")
 LDFLAGS ?= "-X ${PKG}/pkg/smb.driverVersion=${IMAGE_VERSION} -X ${PKG}/pkg/smb.gitCommit=${GIT_COMMIT} -X ${PKG}/pkg/smb.buildDate=${BUILD_DATE} -s -w -extldflags '-static'"
 E2E_HELM_OPTIONS ?= --set image.smb.repository=$(REGISTRY)/$(IMAGE_NAME) --set image.smb.tag=$(IMAGE_VERSION)
+E2E_HELM_OPTIONS += ${EXTRA_HELM_OPTIONS}
 GINKGO_FLAGS = -ginkgo.v
 GO111MODULE = on
 GOPATH ?= $(shell go env GOPATH)
diff --git a/charts/README.md b/charts/README.md
@@ -46,14 +46,19 @@ The following table lists the configurable parameters of the latest SMB CSI Driv
 | `imagePullSecrets`                                | Specify docker-registry secret names as an array           | [] (does not add image pull secrets to deployed pods)             |
 | `serviceAccount.create`                           | whether create service account of csi-smb-controller       | true                                                              |
 | `rbac.create`                                     | whether create rbac of csi-smb-controller                  | true                                                              |
+| `rbac.name`                                       | driver name in rbac role                | true                                                         |
+| `controller.name`                                 | name of driver deployment               | `csi-smb-controller`
 | `controller.replicas`                             | the replicas of csi-smb-controller                                  | 2                                                   |
 | `controller.metricsPort`                          | metrics port of csi-smb-controller                   |29644                                               |
+| `controller.livenessProbe.healthPort `            | health check port for liveness probe                   | `29642` |
 | `controller.logLevel`                             | controller driver log level                                                          |`5`                                                           |
 | `node.metricsPort`                                | metrics port of csi-smb-node                         |29645
 | `node.logLevel`                                   | node driver log level                                                          |`5`                                                           |
 | `linux.enabled`                                   | whether enable linux feature                               | true                                                              |
+| `linux.dsName`                                    | name of driver daemonset on linux                             |`csi-smb-node`                                                         |
 | `linux.kubelet`                                   | configure kubelet directory path on Linux agent node node                  | `/var/lib/kubelet`                                                |
 | `windows.enabled`                                 | whether enable windows feature                             | false                                                             |
+| `windows.dsName`                                  | name of driver daemonset on windows                             |`csi-smb-node-win`                                                         |
 | `windows.kubelet`                                 | configure kubelet directory path on Windows agent node                | `'C:\var\lib\kubelet'`                                            |
 | `windows.image.livenessProbe.repository`          | windows liveness-probe docker image                        | mcr.microsoft.com/oss/kubernetes-csi/livenessprobe                |
 | `windows.image.livenessProbe.tag`                 | windows liveness-probe docker image tag                    | v2.3.0                                 |
@@ -62,7 +67,7 @@ The following table lists the configurable parameters of the latest SMB CSI Driv
 | `windows.image.nodeDriverRegistrar.tag`           | windows csi-node-driver-registrar docker image tag         | v2.2.0                                 |
 | `windows.image.nodeDriverRegistrar.pullPolicy`    | windows csi-node-driver-registrar image pull policy        | IfNotPresent                                                      |
 | `controller.runOnMaster`                          | run controller on master node                              | false                                                             |
-| `node.livenessProbe.healthPort `                  | the health check port for liveness probe                   | `29643` |
+| `node.livenessProbe.healthPort `                  | health check port for liveness probe                   | `29643` |
 | `controller.resources.csiProvisioner.limits.cpu`      | csi-provisioner cpu limits                            | 100m                                                           |
 | `controller.resources.csiProvisioner.limits.memory`   | csi-provisioner memory limits                         | 100Mi                                                          |
 | `controller.resources.csiProvisioner.requests.cpu`    | csi-provisioner cpu requests limits                   | 10m                                                            |
@@ -82,6 +87,8 @@ The following table lists the configurable parameters of the latest SMB CSI Driv
 | `controller.affinity`                                 | controller pod affinity                               | {}                                                             |
 | `controller.nodeSelector`                             | controller pod node selector                          | {}                                                             |
 | `controller.tolerations`                              | controller pod tolerations                            | []                                                             |
+| `node.metricsPort`                                    | metrics port of csi-smb-node                         |`29645`                                                        |
+| `node.livenessProbe.healthPort `                      | health check port for liveness probe                   | `29643` |
 | `node.resources.livenessProbe.limits.cpu`             | liveness-probe cpu limits                             | 100m                                                           |
 | `node.resources.livenessProbe.limits.memory`          | liveness-probe memory limits                          | 100Mi                                                          |
 | `node.resources.livenessProbe.requests.cpu`           | liveness-probe cpu requests limits                    | 10m                                                            |
diff --git a/charts/latest/csi-driver-smb-v1.2.0.tgz b/charts/latest/csi-driver-smb-v1.2.0.tgz
diff --git a/charts/latest/csi-driver-smb/templates/csi-smb-controller.yaml b/charts/latest/csi-driver-smb/templates/csi-smb-controller.yaml
@@ -2,18 +2,18 @@
 kind: Deployment
 apiVersion: apps/v1
 metadata:
-  name: csi-smb-controller
+  name: {{ .Values.controller.name }}
   namespace: {{ .Release.Namespace }}
 {{ include "smb.labels" . | indent 2 }}
 spec:
   replicas: {{ .Values.controller.replicas }}
   selector:
     matchLabels:
-      app: csi-smb-controller
+      app: {{ .Values.controller.name }}
   template:
     metadata:
 {{ include "smb.labels" . | indent 6 }}
-        app: csi-smb-controller
+        app: {{ .Values.controller.name }}
         {{- if .Values.podLabels }}
 {{- toYaml .Values.podLabels | nindent 8 }}
         {{- end }}
@@ -27,7 +27,7 @@ spec:
 {{ toYaml . | indent 8 }}
 {{- end }}
       dnsPolicy: ClusterFirstWithHostNet
-      serviceAccountName: csi-smb-controller-sa
+      serviceAccountName: {{ .Values.serviceAccount.controller }}
       nodeSelector:
 {{- with .Values.controller.nodeSelector }}
 {{ toYaml . | indent 8 }}
@@ -62,7 +62,7 @@ spec:
           args:
             - --csi-address=/csi/csi.sock
             - --probe-timeout=3s
-            - --health-port=29642
+            - --health-port={{ .Values.controller.livenessProbe.healthPort }}
             - --v=2
           imagePullPolicy: {{ .Values.image.livenessProbe.pullPolicy }}
           volumeMounts:
@@ -78,7 +78,7 @@ spec:
             - "--metrics-address=0.0.0.0:{{ .Values.controller.metricsPort }}"
             - "--drivername={{ .Values.driver.name }}"
           ports:
-            - containerPort: 29642
+            - containerPort: {{ .Values.controller.livenessProbe.healthPort }}
               name: healthz
               protocol: TCP
             - containerPort: {{ .Values.controller.metricsPort }}
diff --git a/charts/latest/csi-driver-smb/templates/csi-smb-node-windows.yaml b/charts/latest/csi-driver-smb/templates/csi-smb-node-windows.yaml
@@ -2,17 +2,17 @@
 kind: DaemonSet
 apiVersion: apps/v1
 metadata:
-  name: csi-smb-node-win
+  name: {{ .Values.windows.dsName }}
   namespace: {{ .Release.Namespace }}
 {{ include "smb.labels" . | indent 2 }}
 spec:
   selector:
     matchLabels:
-      app: csi-smb-node-win
+      app: {{ .Values.windows.dsName }}
   template:
     metadata:
 {{ include "smb.labels" . | indent 6 }}
-        app: csi-smb-node-win
+        app: {{ .Values.windows.dsName }}
     spec:
 {{- with .Values.windows.tolerations }}
       tolerations:
@@ -31,7 +31,7 @@ spec:
           args:
             - --csi-address=$(CSI_ENDPOINT)
             - --probe-timeout=3s
-            - --health-port=29643
+            - --health-port={{ .Values.node.livenessProbe.healthPort }}
             - --v=2
           env:
             - name: CSI_ENDPOINT
@@ -80,7 +80,7 @@ spec:
             - --nodeid=$(KUBE_NODE_NAME)
             - "--metrics-address=0.0.0.0:{{ .Values.node.metricsPort }}"
           ports:
-            - containerPort: 29643
+            - containerPort: {{ .Values.node.livenessProbe.healthPort }}
               name: healthz
               protocol: TCP
           livenessProbe:
diff --git a/charts/latest/csi-driver-smb/templates/csi-smb-node.yaml b/charts/latest/csi-driver-smb/templates/csi-smb-node.yaml
@@ -2,17 +2,17 @@
 kind: DaemonSet
 apiVersion: apps/v1
 metadata:
-  name: csi-smb-node
+  name: {{ .Values.linux.dsName }}
   namespace: {{ .Release.Namespace }}
 {{ include "smb.labels" . | indent 2 }}
 spec:
   selector:
     matchLabels:
-      app: csi-smb-node
+      app: {{ .Values.linux.dsName }}
   template:
     metadata:
 {{ include "smb.labels" . | indent 6 }}
-        app: csi-smb-node
+        app: {{ .Values.linux.dsName }}
         {{- if .Values.podLabels }}
 {{- toYaml .Values.podLabels | nindent 8 }}
         {{- end }}
@@ -82,7 +82,7 @@ spec:
             - "--nodeid=$(KUBE_NODE_NAME)"
             - "--metrics-address=0.0.0.0:{{ .Values.node.metricsPort }}"
           ports:
-            - containerPort: 29643
+            - containerPort: {{ .Values.node.livenessProbe.healthPort }}
               name: healthz
               protocol: TCP
           livenessProbe:
diff --git a/charts/latest/csi-driver-smb/templates/rbac-csi-smb-controller.yaml b/charts/latest/csi-driver-smb/templates/rbac-csi-smb-controller.yaml
@@ -3,7 +3,7 @@
 apiVersion: v1
 kind: ServiceAccount
 metadata:
-  name: csi-smb-controller-sa
+  name: {{ .Values.serviceAccount.controller }}
   namespace: {{ .Release.Namespace }}
 {{ include "smb.labels" . | indent 2 }}
 ---
@@ -13,7 +13,7 @@ metadata:
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
-  name: smb-external-provisioner-role
+  name: {{ .Values.rbac.name }}-external-provisioner-role
 {{ include "smb.labels" . | indent 2 }}
 rules:
   - apiGroups: [""]
@@ -45,14 +45,14 @@ rules:
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
-  name: smb-csi-provisioner-binding
+  name: {{ .Values.rbac.name }}-csi-provisioner-binding
 {{ include "smb.labels" . | indent 2 }}
 subjects:
   - kind: ServiceAccount
-    name: csi-smb-controller-sa
+    name: {{ .Values.serviceAccount.controller }}
     namespace: {{ .Release.Namespace }}
 roleRef:
   kind: ClusterRole
-  name: smb-external-provisioner-role
+  name: {{ .Values.rbac.name }}-external-provisioner-role
   apiGroup: rbac.authorization.k8s.io
 {{- end -}}
diff --git a/charts/latest/csi-driver-smb/values.yaml b/charts/latest/csi-driver-smb/values.yaml
@@ -17,17 +17,22 @@ image:
     pullPolicy: IfNotPresent
 
 serviceAccount:
-  create: true
+  create: true # When true, service accounts will be created for you. Set to false if you want to use your own.
+  controller: csi-smb-controller-sa # Name of Service Account to be created or used
 
 rbac:
   create: true
+  name: smb
 
 driver:
   name: smb.csi.k8s.io
 
 controller:
+  name: csi-smb-controller
   replicas: 2
   metricsPort: 29644
+  livenessProbe:
+    healthPort: 29642
   runOnMaster: false
   logLevel: 5
   resources:
@@ -56,12 +61,10 @@ controller:
   nodeSelector: {}
   tolerations:
     - key: "node-role.kubernetes.io/master"
-      operator: "Equal"
-      value: "true"
+      operator: "Exists"
       effect: "NoSchedule"
     - key: "node-role.kubernetes.io/controlplane"
-      operator: "Equal"
-      value: "true"
+      operator: "Exists"
       effect: "NoSchedule"
 
 node:
@@ -96,12 +99,14 @@ node:
 
 linux:
   enabled: true
+  dsName: csi-smb-node # daemonset name
   kubelet: /var/lib/kubelet
   tolerations:
     - operator: "Exists"
 
 windows:
   enabled: false
+  dsName: csi-smb-node-win # daemonset name
   kubelet: 'C:\var\lib\kubelet'
   image:
     livenessProbe:
diff --git a/deploy/csi-smb-controller.yaml b/deploy/csi-smb-controller.yaml
@@ -21,12 +21,10 @@ spec:
       priorityClassName: system-cluster-critical
       tolerations:
         - key: "node-role.kubernetes.io/master"
-          operator: "Equal"
-          value: "true"
+          operator: "Exists"
           effect: "NoSchedule"
         - key: "node-role.kubernetes.io/controlplane"
-          operator: "Equal"
-          value: "true"
+          operator: "Exists"
           effect: "NoSchedule"
       containers:
         - name: csi-provisioner
diff --git a/deploy/example/deployment.yaml b/deploy/example/deployment.yaml
@@ -1,4 +1,16 @@
 ---
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  name: pvc-smb
+spec:
+  accessModes:
+    - ReadWriteMany
+  resources:
+    requests:
+      storage: 10Gi
+  storageClassName: smb
+---
 apiVersion: apps/v1
 kind: Deployment
 metadata:
diff --git a/hack/verify-examples.sh b/hack/verify-examples.sh
@@ -15,21 +15,29 @@
 
 set -euo pipefail
 
+rollout_and_wait() {
+    echo "Applying config \"$1\""
+    trap "echo \"Failed to apply config \\\"$1\\\"\" >&2" err
+
+    APPNAME=$(kubectl apply -f $1 | grep -E "^(:?daemonset|deployment|statefulset|pod)" | awk '{printf $1}')
+    if [[ -n $(expr "${APPNAME}" : "\(daemonset\|deployment\|statefulset\)" || true) ]]; then
+        kubectl rollout status $APPNAME --watch --timeout=5m
+    else
+        kubectl wait "${APPNAME}" --for condition=ready --timeout=5m
+    fi
+}
+
 echo "begin to create deployment examples ..."
 kubectl apply -f deploy/example/storageclass-smb.yaml
-kubectl apply -f deploy/example/pvc-smb.yaml
-kubectl apply -f deploy/example/deployment.yaml
-kubectl apply -f deploy/example/statefulset.yaml
-kubectl apply -f deploy/example/statefulset-nonroot.yaml
-
-echo "sleep 60s ..."
-sleep 60
 
-echo "begin to check pod status ..."
-kubectl get pods -o wide
+EXAMPLES=(\
+    deploy/example/deployment.yaml \
+    deploy/example/statefulset.yaml \
+    deploy/example/statefulset-nonroot.yaml \
+)
 
-kubectl get pods --field-selector status.phase=Running | grep deployment-smb
-kubectl get pods --field-selector status.phase=Running | grep statefulset-smb-0
-kubectl get pods --field-selector status.phase=Running | grep statefulset-smb-nonroot-0
+for EXAMPLE in "${EXAMPLES[@]}"; do
+    rollout_and_wait $EXAMPLE
+done
 
 echo "deployment examples running completed."
diff --git a/test/external-e2e/run.sh b/test/external-e2e/run.sh
@@ -17,6 +17,7 @@
 set -xe
 
 PROJECT_ROOT=$(git rev-parse --show-toplevel)
+DRIVER="test"
 
 install_ginkgo () {
     apt update -y
@@ -28,22 +29,27 @@ setup_e2e_binaries() {
     curl -sL https://storage.googleapis.com/kubernetes-release/release/v1.21.0/kubernetes-test-linux-amd64.tar.gz --output e2e-tests.tar.gz
     tar -xvf e2e-tests.tar.gz && rm e2e-tests.tar.gz
 
-    # install csi driver
-    mkdir -p /tmp/csi && cp deploy/example/storageclass-smb.yaml /tmp/csi/storageclass.yaml
-    make e2e-bootstrap
+    # test on alternative driver name
+    export EXTRA_HELM_OPTIONS=" --set driver.name=$DRIVER.csi.k8s.io --set controller.name=csi-$DRIVER-controller --set linux.dsName=csi-$DRIVER-node --set windows.dsName=csi-$DRIVER-node-win"
+    sed -i "s/smb.csi.k8s.io/$DRIVER.csi.k8s.io/g" deploy/example/storageclass-smb.yaml
     make install-smb-provisioner
+    make e2e-bootstrap
+    sed -i "s/csi-smb-controller/csi-$DRIVER-controller/g" deploy/example/metrics/csi-smb-controller-svc.yaml
     make create-metrics-svc
 }
 
 print_logs() {
+    bash ./hack/verify-examples.sh
     echo "print out driver logs ..."
-    bash ./test/utils/smb_log.sh
+    bash ./test/utils/smb_log.sh $DRIVER
 }
 
 install_ginkgo
 setup_e2e_binaries
 trap print_logs EXIT
 
+mkdir -p /tmp/csi
+cp deploy/example/storageclass-smb.yaml /tmp/csi/storageclass.yaml
 ginkgo -p --progress --v -focus='External.Storage' \
        -skip='\[Disruptive\]|\[Slow\]|unmount after the subpath directory is deleted|support two pods which share the same volume' kubernetes/test/bin/e2e.test  -- \
        -storage.testdriver=$PROJECT_ROOT/test/external-e2e/testdriver.yaml \
diff --git a/test/external-e2e/testdriver.yaml b/test/external-e2e/testdriver.yaml
@@ -5,7 +5,7 @@ ShortName: smb
 StorageClass:
   FromFile: /tmp/csi/storageclass.yaml
 DriverInfo:
-  Name: smb.csi.k8s.io
+  Name: test.csi.k8s.io
   Capabilities:
     persistence: true
     exec: true
diff --git a/test/utils/smb_log.sh b/test/utils/smb_log.sh
