[WIP] feat: support SMB mount with managed identity

charts/latest/azurefile-csi-driver/templates/csi-azurefile-node.yaml

@@ -96,11 +96,18 @@ spec:
               value: "{{ .Values.linux.kubelet }}"
             - name: MIGRATE_K8S_REPO
               value: "{{ .Values.node.azurefileProxy.migrateK8sRepo }}"
+            - name: ENABLE_MI_AUTH
+              value: "{{ .Values.node.enableManagedIdentityAuth }}"
           volumeMounts:
             - name: host-usr
               mountPath: /host/usr
             - name: host-etc
               mountPath: /host/etc
+            {{- if .Values.node.enableManagedIdentityAuth }}
+            - name: mountpoint-dir
+              mountPath: /var/lib/kubelet/
+              mountPropagation: Bidirectional
+            {{- end }}
       containers:
         - name: liveness-probe
           volumeMounts:
@@ -140,6 +147,31 @@ spec:
             - name: registration-dir
               mountPath: /registration
           resources: {{- toYaml .Values.linux.resources.nodeDriverRegistrar | nindent 12 }}
+{{- if .Values.node.enableManagedIdentityAuth }}
+        - name: azfilesrefresh
+{{- if hasPrefix "/" .Values.image.azurefile.repository }}
+          image: "{{ .Values.image.baseRepo }}{{ .Values.image.azurefile.repository }}:{{ .Values.image.azurefile.tag }}"
+{{- else }}
+          image: "{{ .Values.image.azurefile.repository }}:{{ .Values.image.azurefile.tag }}"
+{{- end }}
+          imagePullPolicy: {{ .Values.image.azurefile.pullPolicy }}
+          command:
+            - "azfilesrefresh"
+          securityContext:
+            privileged: true
+            capabilities:
+              drop:
+                - ALL
+          volumeMounts:
+            - mountPath: /var/lib/kubelet/
+              mountPropagation: Bidirectional
+              name: mountpoint-dir
+            - name: host-etc
+              mountPath: /etc
+            - name: log-dir
+              mountPath: /var/log/
+          resources: {{- toYaml .Values.linux.resources.azfilesrefresh | nindent 12 }}
+{{- end }}
         - name: azurefile
 {{- if hasPrefix "/" .Values.image.azurefile.repository }}
           image: "{{ .Values.image.baseRepo }}{{ .Values.image.azurefile.repository }}:{{ .Values.image.azurefile.tag }}"
@@ -228,6 +260,10 @@ spec:
             - mountPath: /run/kata-containers/shared/direct-volumes
               name: kata-direct-volumes
             {{- end }}
+            {{- if .Values.node.enableManagedIdentityAuth }}
+            - name: log-dir
+              mountPath: /var/log/
+            {{- end }}
           resources: {{- toYaml .Values.linux.resources.azurefile | nindent 12 }}
       volumes:
         - name: host-usr
@@ -270,4 +306,10 @@ spec:
             path: /run/kata-containers/shared/direct-volumes/
             type: DirectoryOrCreate
         {{- end }}
+        {{- if .Values.node.enableManagedIdentityAuth }}
+        - hostPath:
+            path: /var/log/
+            type: DirectoryOrCreate
+          name: log-dir
+        {{- end }}
 {{- end -}}

charts/latest/azurefile-csi-driver/values.yaml

@@ -88,6 +88,13 @@ controller:
       requests:
         cpu: 10m
         memory: 20Mi
+    azfilesrefresh:
+      limits:
+        cpu: 1
+        memory: 100Mi
+      requests:
+        cpu: 10m
+        memory: 20Mi
     azurefile:
       limits:
         cpu: 2
@@ -120,6 +127,7 @@ node:
   allowEmptyCloudConfig: true
   allowInlineVolumeKeyAccessWithIdentity: false
   enableKataCCMount: false
+  enableManagedIdentityAuth: true
   metricsPort: 29615
   livenessProbe:
     healthPort: 29613

deploy/csi-azurefile-node.yaml

@@ -58,11 +58,16 @@ spec:
               value: "true"
             - name: INSTALL_AZNFS_MOUNT
               value: "true"
+            - name: ENABLE_MI_AUTH
+              value: "true"
           volumeMounts:
             - name: host-usr
               mountPath: /host/usr
             - name: host-etc
               mountPath: /host/etc
+            - name: mountpoint-dir
+              mountPath: /var/lib/kubelet/
+              mountPropagation: Bidirectional
       containers:
         - name: liveness-probe
           volumeMounts:
@@ -163,12 +168,40 @@ spec:
               name: device-dir
             - mountPath: /run/kata-containers/shared/direct-volumes
               name: kata-direct-volumes
+            - name: host-etc
+              mountPath: /etc
+            - name: log-dir
+              mountPath: /var/log/
           resources:
             limits:
               memory: 400Mi
             requests:
               cpu: 10m
               memory: 20Mi
+        - name: azfilesrefresh
+          image: mcr.microsoft.com/k8s/csi/azurefile-csi:latest
+          imagePullPolicy: IfNotPresent
+          command:
+            - "azfilesrefresh"
+          securityContext:
+            privileged: true
+            capabilities:
+              drop:
+                - ALL
+          volumeMounts:
+            - mountPath: /var/lib/kubelet/
+              mountPropagation: Bidirectional
+              name: mountpoint-dir
+            - name: host-etc
+              mountPath: /etc
+            - name: log-dir
+              mountPath: /var/log/
+          resources:
+            limits:
+              memory: 100Mi
+            requests:
+              cpu: 10m
+              memory: 20Mi
       volumes:
         - name: host-usr
           hostPath:
@@ -200,4 +233,8 @@ spec:
           hostPath:
             path: /run/kata-containers/shared/direct-volumes/
             type: DirectoryOrCreate
+        - hostPath:
+            path: /var/log/
+            type: DirectoryOrCreate
+          name: log-dir
 ---

pkg/azurefile-proxy/init.sh

@@ -17,16 +17,28 @@
 set -xe
 
 MIGRATE_K8S_REPO=${MIGRATE_K8S_REPO:-false}
+ENABLE_MI_AUTH=${ENABLE_MI_AUTH:-true}
 
 KUBELET_PATH=${KUBELET_PATH:-/var/lib/kubelet}
 if [ "$KUBELET_PATH" != "/var/lib/kubelet" ];then
   echo "kubelet path is $KUBELET_PATH, update azurefile-proxy.service...."
   sed -i "s#--azurefile-proxy-endpoint[^ ]*#--azurefile-proxy-endpoint=unix:/${KUBELET_PATH}/plugins/file.csi.azure.com/azurefile-proxy.sock#" /azurefile-proxy/azurefile-proxy.service
   echo "azurefile-proxy endpoint is updated to unix:/$KUBELET_PATH/plugins/file.csi.azure.com/azurefile-proxy.sock"
-fi 
+fi
 
 HOST_CMD="nsenter --mount=/proc/1/ns/mnt"
 
+if [ "$ENABLE_MI_AUTH" = "true" ];then
+  echo "set up /etc/krb5.conf on host"
+  printf '[libdefaults]\ndefault_ccache_name = FILE:/var/lib/kubelet/kerberos/krb5cc_%s\n' "%{uid}" > /host/etc/krb5.conf
+
+  mkdir -p /var/lib/kubelet/kerberos
+
+  echo "set up /etc/azfilesauth/config.yaml on host"
+  mkdir -p /host/etc/azfilesauth
+  printf 'USER_UID: 0\nKRB5_CC_NAME: /var/lib/kubelet/kerberos/krb5cc_0\n' > /host/etc/azfilesauth/config.yaml
+fi
+
 DISTRIBUTION=$($HOST_CMD cat /etc/os-release | grep ^ID= | cut -d'=' -f2 | tr -d '"')
 ARCH=$($HOST_CMD uname -m)
 echo "Linux distribution: $DISTRIBUTION, Arch: $ARCH"

pkg/azurefile/azurefile.go

@@ -166,6 +166,7 @@ const (
 	defaultConfidentialContainerLabel = "kubernetes.azure.com/kata-cc-isolation"
 	runtimeClassHandlerField          = "runtimeclasshandler"
 	defaultRuntimeClassHandler        = "kata-cc"
+	mountWithManagedIdentityField     = "mountwithmanagedidentity"
 
 	accountNotProvisioned = "StorageAccountIsNotProvisioned"
 	// this is a workaround fix for 429 throttling issue, will update cloud provider for better fix later
@@ -782,7 +783,7 @@ func (d *Driver) GetAccountInfo(ctx context.Context, volumeID string, secrets, r
 
 	var protocol, accountKey, secretName, pvcNamespace string
 	// getAccountKeyFromSecret indicates whether get account key only from k8s secret
-	var getAccountKeyFromSecret, getLatestAccountKey bool
+	var getAccountKeyFromSecret, getLatestAccountKey, mountWithManagedIdentity bool
 	var clientID, tenantID, serviceAccountToken string
 
 	for k, v := range reqContext {
@@ -815,6 +816,10 @@ func (d *Driver) GetAccountInfo(ctx context.Context, volumeID string, secrets, r
 			}
 		case clientIDField:
 			clientID = v
+		case mountWithManagedIdentityField:
+			if mountWithManagedIdentity, err = strconv.ParseBool(v); err != nil {
+				return rgName, accountName, accountKey, fileShareName, diskName, subsID, fmt.Errorf("invalid %s: %s in volume context", mountWithManagedIdentityField, v)
+			}
 		case tenantIDField:
 			tenantID = v
 		case strings.ToLower(serviceAccountTokenField):
@@ -844,7 +849,11 @@ func (d *Driver) GetAccountInfo(ctx context.Context, volumeID string, secrets, r
 		}
 	}
 
-	// if client id is specified, we only use service account token to get account key
+	if mountWithManagedIdentity {
+		klog.V(2).Infof("mountWithManagedIdentity is true, use managed identity auth")
+		return rgName, accountName, accountKey, fileShareName, diskName, subsID, nil
+	}
+
 	if clientID != "" {
 		klog.V(2).Infof("clientID(%s) is specified, use service account token to get account key", clientID)
 		accountKey, err := d.cloud.GetStorageAccesskeyFromServiceAccountToken(ctx, subsID, accountName, rgName, clientID, tenantID, serviceAccountToken)

pkg/azurefile/controllerserver.go

@@ -227,6 +227,7 @@ func (d *Driver) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest)
 			fileShareNameReplaceMap[pvNameMetadata] = v
 		case serverNameField:
 		case folderNameField:
+		case mountWithManagedIdentityField:
 		case clientIDField:
 		case confidentialContainerLabelField:
 		case runtimeClassHandlerField:

pkg/azurefile/nodeserver.go

@@ -76,8 +76,7 @@ func (d *Driver) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolu
 	mountPermissions := d.mountPermissions
 	context := req.GetVolumeContext()
 	if context != nil {
-		// token request
-		if getValueInMap(context, serviceAccountTokenField) != "" && getValueInMap(context, clientIDField) != "" {
+		if !strings.EqualFold(getValueInMap(context, mountWithManagedIdentityField), trueValue) && getValueInMap(context, serviceAccountTokenField) != "" && getValueInMap(context, clientIDField) != "" {
 			klog.V(2).Infof("NodePublishVolume: volume(%s) mount on %s with service account token, clientID: %s", volumeID, target, getValueInMap(context, clientIDField))
 			_, err := d.NodeStageVolume(ctx, &csi.NodeStageVolumeRequest{
 				StagingTargetPath: target,
@@ -248,7 +247,7 @@ func (d *Driver) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRe
 	volumeID := req.GetVolumeId()
 	context := req.GetVolumeContext()
 
-	if getValueInMap(context, clientIDField) != "" && getValueInMap(context, serviceAccountTokenField) == "" {
+	if getValueInMap(context, clientIDField) != "" && !strings.EqualFold(getValueInMap(context, mountWithManagedIdentityField), trueValue) && getValueInMap(context, serviceAccountTokenField) == "" {
 		klog.V(2).Infof("Skip NodeStageVolume for volume(%s) since clientID %s is provided but service account token is empty", volumeID, getValueInMap(context, clientIDField))
 		return &csi.NodeStageVolumeResponse{}, nil
 	}
@@ -277,9 +276,8 @@ func (d *Driver) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRe
 	}
 	// don't respect fsType from req.GetVolumeCapability().GetMount().GetFsType()
 	// since it's ext4 by default on Linux
-	var fsType, server, protocol, ephemeralVolMountOptions, storageEndpointSuffix, folderName string
-	var ephemeralVol bool
-	var encryptInTransit bool
+	var fsType, server, protocol, ephemeralVolMountOptions, storageEndpointSuffix, folderName, clientID string
+	var ephemeralVol, encryptInTransit, mountWithManagedIdentity bool
 	fileShareNameReplaceMap := map[string]string{}
 
 	mountPermissions := d.mountPermissions
@@ -313,7 +311,6 @@ func (d *Driver) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRe
 			fileShareNameReplaceMap[pvNameMetadata] = v
 		case mountPermissionsField:
 			if v != "" {
-				var err error
 				var perm uint64
 				if perm, err = strconv.ParseUint(v, 8, 32); err != nil {
 					return nil, status.Errorf(codes.InvalidArgument, "invalid mountPermissions %s", v)
@@ -325,11 +322,17 @@ func (d *Driver) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRe
 				}
 			}
 		case encryptInTransitField:
-			var err error
 			encryptInTransit, err = strconv.ParseBool(v)
 			if err != nil {
 				return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("Volume context property %q must be a boolean value: %v", k, err))
 			}
+		case mountWithManagedIdentityField:
+			mountWithManagedIdentity, err = strconv.ParseBool(v)
+			if err != nil {
+				return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("Volume context property %q must be a boolean value: %v", k, err))
+			}
+		case clientIDField:
+			clientID = v
 		}
 	}
 
@@ -394,18 +397,29 @@ func (d *Driver) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRe
 		mountOptions = util.JoinMountOptions(mountFlags, []string{"vers=4,minorversion=1,sec=sys"})
 		mountOptions = appendDefaultNfsMountOptions(mountOptions, d.appendNoResvPortOption, d.appendActimeoOption)
 	} else {
-		if accountName == "" || accountKey == "" {
-			return nil, status.Errorf(codes.Internal, "accountName(%s) or accountKey is empty", accountName)
-		}
-		if runtime.GOOS == "windows" {
-			mountOptions = []string{fmt.Sprintf("AZURE\\%s", accountName)}
-			sensitiveMountOptions = []string{accountKey}
+		if mountWithManagedIdentity && runtime.GOOS != "windows" {
+			if clientID == "" {
+				clientID = d.cloud.Config.AzureAuthConfig.UserAssignedIdentityID
+			}
+			sensitiveMountOptions = []string{"sec=krb5,cruid=0,upcall_target=mount", fmt.Sprintf("username=%s", clientID)}
+			klog.V(2).Infof("using managed identity %s for volume %s with mount options: %v", clientID, volumeID, sensitiveMountOptions)
 		} else {
-			if err := os.MkdirAll(targetPath, os.FileMode(mountPermissions)); err != nil {
-				return nil, status.Error(codes.Internal, fmt.Sprintf("MkdirAll %s failed with error: %v", targetPath, err))
+			if accountName == "" || accountKey == "" {
+				return nil, status.Errorf(codes.Internal, "accountName(%s) or accountKey is empty", accountName)
 			}
-			// parameters suggested by https://azure.microsoft.com/en-us/documentation/articles/storage-how-to-use-files-linux/
-			sensitiveMountOptions = []string{fmt.Sprintf("username=%s,password=%s", accountName, accountKey)}
+			if runtime.GOOS == "windows" {
+				mountOptions = []string{fmt.Sprintf("AZURE\\%s", accountName)}
+				sensitiveMountOptions = []string{accountKey}
+			} else {
+				if err := os.MkdirAll(targetPath, os.FileMode(mountPermissions)); err != nil {
+					return nil, status.Error(codes.Internal, fmt.Sprintf("MkdirAll %s failed with error: %v", targetPath, err))
+				}
+				// parameters suggested by https://azure.microsoft.com/en-us/documentation/articles/storage-how-to-use-files-linux/
+				sensitiveMountOptions = []string{fmt.Sprintf("username=%s,password=%s", accountName, accountKey)}
+			}
+		}
+
+		if runtime.GOOS != "windows" {
 			if ephemeralVol {
 				cifsMountFlags = util.JoinMountOptions(cifsMountFlags, strings.Split(ephemeralVolMountOptions, ","))
 			}
@@ -449,6 +463,11 @@ func (d *Driver) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRe
 			klog.V(2).Infof("mount with proxy succeeded for %s", cifsMountPath)
 		} else {
 			execFunc := func() error {
+				if mountWithManagedIdentity && protocol != nfs && runtime.GOOS != "windows" {
+					if out, err := setCredentialCache(server, clientID); err != nil {
+						return fmt.Errorf("setCredentialCache failed for %s with error: %v, output: %s", server, err, out)
+					}
+				}
 				return SMBMount(d.mounter, source, cifsMountPath, mountFsType, mountOptions, sensitiveMountOptions)
 			}
 			timeoutFunc := func() error { return fmt.Errorf("time out") }

pkg/azurefile/utils.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"os/exec"
 	"regexp"
 	"strconv"
 	"strings"
@@ -372,3 +373,10 @@ func removeOptionIfExists(options []string, removeOption string) ([]string, bool
 	}
 	return options, false
 }
+
+func setCredentialCache(server, clientID string) ([]byte, error) {
+	cmd := exec.Command("azfilesauthmanager", "set", "https://"+server, "--imds-client-id", clientID)
+	cmd.Env = append(os.Environ(), cmd.Env...)
+	klog.V(2).Infof("Executing command: %q", cmd.String())
+	return cmd.CombinedOutput()
+}