diff --git a/charts/latest/azurefile-csi-driver-v0.0.0.tgz b/charts/latest/azurefile-csi-driver-v0.0.0.tgz index f36df1f65d..fded5482f1 100644 Binary files a/charts/latest/azurefile-csi-driver-v0.0.0.tgz and b/charts/latest/azurefile-csi-driver-v0.0.0.tgz differ diff --git a/charts/latest/azurefile-csi-driver/templates/csi-azurefile-node.yaml b/charts/latest/azurefile-csi-driver/templates/csi-azurefile-node.yaml index 44f8edf36b..c6a8c3e7c3 100644 --- a/charts/latest/azurefile-csi-driver/templates/csi-azurefile-node.yaml +++ b/charts/latest/azurefile-csi-driver/templates/csi-azurefile-node.yaml @@ -96,11 +96,18 @@ spec: value: "{{ .Values.linux.kubelet }}" - name: MIGRATE_K8S_REPO value: "{{ .Values.node.azurefileProxy.migrateK8sRepo }}" + - name: ENABLE_MI_AUTH + value: "{{ .Values.node.enableManagedIdentityAuth }}" volumeMounts: - name: host-usr mountPath: /host/usr - name: host-etc mountPath: /host/etc + {{- if .Values.node.enableManagedIdentityAuth }} + - name: mountpoint-dir + mountPath: /var/lib/kubelet/ + mountPropagation: Bidirectional + {{- end }} containers: - name: liveness-probe volumeMounts: @@ -140,6 +147,31 @@ spec: - name: registration-dir mountPath: /registration resources: {{- toYaml .Values.linux.resources.nodeDriverRegistrar | nindent 12 }} +{{- if .Values.node.enableManagedIdentityAuth }} + - name: azfilesrefresh +{{- if hasPrefix "/" .Values.image.azurefile.repository }} + image: "{{ .Values.image.baseRepo }}{{ .Values.image.azurefile.repository }}:{{ .Values.image.azurefile.tag }}" +{{- else }} + image: "{{ .Values.image.azurefile.repository }}:{{ .Values.image.azurefile.tag }}" +{{- end }} + imagePullPolicy: {{ .Values.image.azurefile.pullPolicy }} + command: + - "azfilesrefresh" + securityContext: + privileged: true + capabilities: + drop: + - ALL + volumeMounts: + - mountPath: /var/lib/kubelet/ + mountPropagation: Bidirectional + name: mountpoint-dir + - name: host-etc + mountPath: /etc + - name: log-dir + mountPath: /var/log/ + resources: {{- toYaml .Values.linux.resources.azfilesrefresh | nindent 12 }} +{{- end }} - name: azurefile {{- if hasPrefix "/" .Values.image.azurefile.repository }} image: "{{ .Values.image.baseRepo }}{{ .Values.image.azurefile.repository }}:{{ .Values.image.azurefile.tag }}" @@ -228,6 +260,10 @@ spec: - mountPath: /run/kata-containers/shared/direct-volumes name: kata-direct-volumes {{- end }} + {{- if .Values.node.enableManagedIdentityAuth }} + - name: log-dir + mountPath: /var/log/ + {{- end }} resources: {{- toYaml .Values.linux.resources.azurefile | nindent 12 }} volumes: - name: host-usr @@ -270,4 +306,10 @@ spec: path: /run/kata-containers/shared/direct-volumes/ type: DirectoryOrCreate {{- end }} + {{- if .Values.node.enableManagedIdentityAuth }} + - hostPath: + path: /var/log/ + type: DirectoryOrCreate + name: log-dir + {{- end }} {{- end -}} diff --git a/charts/latest/azurefile-csi-driver/values.yaml b/charts/latest/azurefile-csi-driver/values.yaml index fe8972a0a2..860aa44057 100644 --- a/charts/latest/azurefile-csi-driver/values.yaml +++ b/charts/latest/azurefile-csi-driver/values.yaml @@ -88,6 +88,13 @@ controller: requests: cpu: 10m memory: 20Mi + azfilesrefresh: + limits: + cpu: 1 + memory: 100Mi + requests: + cpu: 10m + memory: 20Mi azurefile: limits: cpu: 2 @@ -120,6 +127,7 @@ node: allowEmptyCloudConfig: true allowInlineVolumeKeyAccessWithIdentity: false enableKataCCMount: false + enableManagedIdentityAuth: true metricsPort: 29615 livenessProbe: healthPort: 29613 diff --git a/deploy/csi-azurefile-node.yaml b/deploy/csi-azurefile-node.yaml index 9934fec29f..189b5267ef 100644 --- a/deploy/csi-azurefile-node.yaml +++ b/deploy/csi-azurefile-node.yaml @@ -58,11 +58,16 @@ spec: value: "true" - name: INSTALL_AZNFS_MOUNT value: "true" + - name: ENABLE_MI_AUTH + value: "true" volumeMounts: - name: host-usr mountPath: /host/usr - name: host-etc mountPath: /host/etc + - name: mountpoint-dir + mountPath: /var/lib/kubelet/ + mountPropagation: Bidirectional containers: - name: liveness-probe volumeMounts: @@ -163,12 +168,40 @@ spec: name: device-dir - mountPath: /run/kata-containers/shared/direct-volumes name: kata-direct-volumes + - name: host-etc + mountPath: /etc + - name: log-dir + mountPath: /var/log/ resources: limits: memory: 400Mi requests: cpu: 10m memory: 20Mi + - name: azfilesrefresh + image: mcr.microsoft.com/k8s/csi/azurefile-csi:latest + imagePullPolicy: IfNotPresent + command: + - "azfilesrefresh" + securityContext: + privileged: true + capabilities: + drop: + - ALL + volumeMounts: + - mountPath: /var/lib/kubelet/ + mountPropagation: Bidirectional + name: mountpoint-dir + - name: host-etc + mountPath: /etc + - name: log-dir + mountPath: /var/log/ + resources: + limits: + memory: 100Mi + requests: + cpu: 10m + memory: 20Mi volumes: - name: host-usr hostPath: @@ -200,4 +233,8 @@ spec: hostPath: path: /run/kata-containers/shared/direct-volumes/ type: DirectoryOrCreate + - hostPath: + path: /var/log/ + type: DirectoryOrCreate + name: log-dir --- diff --git a/pkg/azurefile-proxy/init.sh b/pkg/azurefile-proxy/init.sh index 9b76405926..e8e967a65e 100644 --- a/pkg/azurefile-proxy/init.sh +++ b/pkg/azurefile-proxy/init.sh @@ -17,16 +17,28 @@ set -xe MIGRATE_K8S_REPO=${MIGRATE_K8S_REPO:-false} +ENABLE_MI_AUTH=${ENABLE_MI_AUTH:-true} KUBELET_PATH=${KUBELET_PATH:-/var/lib/kubelet} if [ "$KUBELET_PATH" != "/var/lib/kubelet" ];then echo "kubelet path is $KUBELET_PATH, update azurefile-proxy.service...." sed -i "s#--azurefile-proxy-endpoint[^ ]*#--azurefile-proxy-endpoint=unix:/${KUBELET_PATH}/plugins/file.csi.azure.com/azurefile-proxy.sock#" /azurefile-proxy/azurefile-proxy.service echo "azurefile-proxy endpoint is updated to unix:/$KUBELET_PATH/plugins/file.csi.azure.com/azurefile-proxy.sock" -fi +fi HOST_CMD="nsenter --mount=/proc/1/ns/mnt" +if [ "$ENABLE_MI_AUTH" = "true" ];then + echo "set up /etc/krb5.conf on host" + printf '[libdefaults]\ndefault_ccache_name = FILE:/var/lib/kubelet/kerberos/krb5cc_%s\n' "%{uid}" > /host/etc/krb5.conf + + mkdir -p /var/lib/kubelet/kerberos + + echo "set up /etc/azfilesauth/config.yaml on host" + mkdir -p /host/etc/azfilesauth + printf 'USER_UID: 0\nKRB5_CC_NAME: /var/lib/kubelet/kerberos/krb5cc_0\n' > /host/etc/azfilesauth/config.yaml +fi + DISTRIBUTION=$($HOST_CMD cat /etc/os-release | grep ^ID= | cut -d'=' -f2 | tr -d '"') ARCH=$($HOST_CMD uname -m) echo "Linux distribution: $DISTRIBUTION, Arch: $ARCH" diff --git a/pkg/azurefile/azurefile.go b/pkg/azurefile/azurefile.go index 08e1dfda7f..04d53f6b85 100644 --- a/pkg/azurefile/azurefile.go +++ b/pkg/azurefile/azurefile.go @@ -166,6 +166,7 @@ const ( defaultConfidentialContainerLabel = "kubernetes.azure.com/kata-cc-isolation" runtimeClassHandlerField = "runtimeclasshandler" defaultRuntimeClassHandler = "kata-cc" + mountWithManagedIdentityField = "mountwithmanagedidentity" accountNotProvisioned = "StorageAccountIsNotProvisioned" // this is a workaround fix for 429 throttling issue, will update cloud provider for better fix later @@ -782,7 +783,7 @@ func (d *Driver) GetAccountInfo(ctx context.Context, volumeID string, secrets, r var protocol, accountKey, secretName, pvcNamespace string // getAccountKeyFromSecret indicates whether get account key only from k8s secret - var getAccountKeyFromSecret, getLatestAccountKey bool + var getAccountKeyFromSecret, getLatestAccountKey, mountWithManagedIdentity bool var clientID, tenantID, serviceAccountToken string for k, v := range reqContext { @@ -815,6 +816,10 @@ func (d *Driver) GetAccountInfo(ctx context.Context, volumeID string, secrets, r } case clientIDField: clientID = v + case mountWithManagedIdentityField: + if mountWithManagedIdentity, err = strconv.ParseBool(v); err != nil { + return rgName, accountName, accountKey, fileShareName, diskName, subsID, fmt.Errorf("invalid %s: %s in volume context", mountWithManagedIdentityField, v) + } case tenantIDField: tenantID = v case strings.ToLower(serviceAccountTokenField): @@ -844,7 +849,11 @@ func (d *Driver) GetAccountInfo(ctx context.Context, volumeID string, secrets, r } } - // if client id is specified, we only use service account token to get account key + if mountWithManagedIdentity { + klog.V(2).Infof("mountWithManagedIdentity is true, use managed identity auth") + return rgName, accountName, accountKey, fileShareName, diskName, subsID, nil + } + if clientID != "" { klog.V(2).Infof("clientID(%s) is specified, use service account token to get account key", clientID) accountKey, err := d.cloud.GetStorageAccesskeyFromServiceAccountToken(ctx, subsID, accountName, rgName, clientID, tenantID, serviceAccountToken) diff --git a/pkg/azurefile/controllerserver.go b/pkg/azurefile/controllerserver.go index 95bc1320d5..6eaaafc33f 100644 --- a/pkg/azurefile/controllerserver.go +++ b/pkg/azurefile/controllerserver.go @@ -227,6 +227,7 @@ func (d *Driver) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) fileShareNameReplaceMap[pvNameMetadata] = v case serverNameField: case folderNameField: + case mountWithManagedIdentityField: case clientIDField: case confidentialContainerLabelField: case runtimeClassHandlerField: diff --git a/pkg/azurefile/nodeserver.go b/pkg/azurefile/nodeserver.go index 89e65c690a..6149a9132d 100644 --- a/pkg/azurefile/nodeserver.go +++ b/pkg/azurefile/nodeserver.go @@ -76,8 +76,7 @@ func (d *Driver) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolu mountPermissions := d.mountPermissions context := req.GetVolumeContext() if context != nil { - // token request - if getValueInMap(context, serviceAccountTokenField) != "" && getValueInMap(context, clientIDField) != "" { + if !strings.EqualFold(getValueInMap(context, mountWithManagedIdentityField), trueValue) && getValueInMap(context, serviceAccountTokenField) != "" && getValueInMap(context, clientIDField) != "" { klog.V(2).Infof("NodePublishVolume: volume(%s) mount on %s with service account token, clientID: %s", volumeID, target, getValueInMap(context, clientIDField)) _, err := d.NodeStageVolume(ctx, &csi.NodeStageVolumeRequest{ StagingTargetPath: target, @@ -248,7 +247,7 @@ func (d *Driver) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRe volumeID := req.GetVolumeId() context := req.GetVolumeContext() - if getValueInMap(context, clientIDField) != "" && getValueInMap(context, serviceAccountTokenField) == "" { + if getValueInMap(context, clientIDField) != "" && !strings.EqualFold(getValueInMap(context, mountWithManagedIdentityField), trueValue) && getValueInMap(context, serviceAccountTokenField) == "" { klog.V(2).Infof("Skip NodeStageVolume for volume(%s) since clientID %s is provided but service account token is empty", volumeID, getValueInMap(context, clientIDField)) return &csi.NodeStageVolumeResponse{}, nil } @@ -277,9 +276,8 @@ func (d *Driver) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRe } // don't respect fsType from req.GetVolumeCapability().GetMount().GetFsType() // since it's ext4 by default on Linux - var fsType, server, protocol, ephemeralVolMountOptions, storageEndpointSuffix, folderName string - var ephemeralVol bool - var encryptInTransit bool + var fsType, server, protocol, ephemeralVolMountOptions, storageEndpointSuffix, folderName, clientID string + var ephemeralVol, encryptInTransit, mountWithManagedIdentity bool fileShareNameReplaceMap := map[string]string{} mountPermissions := d.mountPermissions @@ -313,7 +311,6 @@ func (d *Driver) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRe fileShareNameReplaceMap[pvNameMetadata] = v case mountPermissionsField: if v != "" { - var err error var perm uint64 if perm, err = strconv.ParseUint(v, 8, 32); err != nil { return nil, status.Errorf(codes.InvalidArgument, "invalid mountPermissions %s", v) @@ -325,11 +322,17 @@ func (d *Driver) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRe } } case encryptInTransitField: - var err error encryptInTransit, err = strconv.ParseBool(v) if err != nil { return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("Volume context property %q must be a boolean value: %v", k, err)) } + case mountWithManagedIdentityField: + mountWithManagedIdentity, err = strconv.ParseBool(v) + if err != nil { + return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("Volume context property %q must be a boolean value: %v", k, err)) + } + case clientIDField: + clientID = v } } @@ -394,18 +397,29 @@ func (d *Driver) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRe mountOptions = util.JoinMountOptions(mountFlags, []string{"vers=4,minorversion=1,sec=sys"}) mountOptions = appendDefaultNfsMountOptions(mountOptions, d.appendNoResvPortOption, d.appendActimeoOption) } else { - if accountName == "" || accountKey == "" { - return nil, status.Errorf(codes.Internal, "accountName(%s) or accountKey is empty", accountName) - } - if runtime.GOOS == "windows" { - mountOptions = []string{fmt.Sprintf("AZURE\\%s", accountName)} - sensitiveMountOptions = []string{accountKey} + if mountWithManagedIdentity && runtime.GOOS != "windows" { + if clientID == "" { + clientID = d.cloud.Config.AzureAuthConfig.UserAssignedIdentityID + } + sensitiveMountOptions = []string{"sec=krb5,cruid=0,upcall_target=mount", fmt.Sprintf("username=%s", clientID)} + klog.V(2).Infof("using managed identity %s for volume %s with mount options: %v", clientID, volumeID, sensitiveMountOptions) } else { - if err := os.MkdirAll(targetPath, os.FileMode(mountPermissions)); err != nil { - return nil, status.Error(codes.Internal, fmt.Sprintf("MkdirAll %s failed with error: %v", targetPath, err)) + if accountName == "" || accountKey == "" { + return nil, status.Errorf(codes.Internal, "accountName(%s) or accountKey is empty", accountName) } - // parameters suggested by https://azure.microsoft.com/en-us/documentation/articles/storage-how-to-use-files-linux/ - sensitiveMountOptions = []string{fmt.Sprintf("username=%s,password=%s", accountName, accountKey)} + if runtime.GOOS == "windows" { + mountOptions = []string{fmt.Sprintf("AZURE\\%s", accountName)} + sensitiveMountOptions = []string{accountKey} + } else { + if err := os.MkdirAll(targetPath, os.FileMode(mountPermissions)); err != nil { + return nil, status.Error(codes.Internal, fmt.Sprintf("MkdirAll %s failed with error: %v", targetPath, err)) + } + // parameters suggested by https://azure.microsoft.com/en-us/documentation/articles/storage-how-to-use-files-linux/ + sensitiveMountOptions = []string{fmt.Sprintf("username=%s,password=%s", accountName, accountKey)} + } + } + + if runtime.GOOS != "windows" { if ephemeralVol { cifsMountFlags = util.JoinMountOptions(cifsMountFlags, strings.Split(ephemeralVolMountOptions, ",")) } @@ -449,6 +463,11 @@ func (d *Driver) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRe klog.V(2).Infof("mount with proxy succeeded for %s", cifsMountPath) } else { execFunc := func() error { + if mountWithManagedIdentity && protocol != nfs && runtime.GOOS != "windows" { + if out, err := setCredentialCache(server, clientID); err != nil { + return fmt.Errorf("setCredentialCache failed for %s with error: %v, output: %s", server, err, out) + } + } return SMBMount(d.mounter, source, cifsMountPath, mountFsType, mountOptions, sensitiveMountOptions) } timeoutFunc := func() error { return fmt.Errorf("time out") } diff --git a/pkg/azurefile/utils.go b/pkg/azurefile/utils.go index 5d669fba1c..eb15708979 100644 --- a/pkg/azurefile/utils.go +++ b/pkg/azurefile/utils.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "os" + "os/exec" "regexp" "strconv" "strings" @@ -372,3 +373,10 @@ func removeOptionIfExists(options []string, removeOption string) ([]string, bool } return options, false } + +func setCredentialCache(server, clientID string) ([]byte, error) { + cmd := exec.Command("azfilesauthmanager", "set", "https://"+server, "--imds-client-id", clientID) + cmd.Env = append(os.Environ(), cmd.Env...) + klog.V(2).Infof("Executing command: %q", cmd.String()) + return cmd.CombinedOutput() +} diff --git a/pkg/azurefileplugin/Dockerfile b/pkg/azurefileplugin/Dockerfile index f13b1a059b..901f80c95a 100644 --- a/pkg/azurefileplugin/Dockerfile +++ b/pkg/azurefileplugin/Dockerfile @@ -22,6 +22,7 @@ ARG ARCH RUN apt update \ && apt install -y curl \ + && curl -Lso /tmp/azfilesauth_amd64.deb https://raw.githubusercontent.com/andyzhangx/demo/refs/heads/master/aks/azfilesauth_1.0-4_${ARCH}.deb \ && curl -Ls https://github.com/Azure/azure-storage-azcopy/releases/download/v10.30.0/azcopy_linux_${ARCH}_10.30.0.tar.gz \ | tar xvzf - --strip-components=1 -C /usr/local/bin/ --wildcards "*/azcopy" @@ -33,16 +34,22 @@ ARG binary=./_output/${ARCH}/azurefileplugin COPY ${binary} /azurefileplugin COPY --from=builder --chown=root:root /usr/local/bin/azcopy /usr/local/bin/azcopy -RUN apt update && apt upgrade -y && apt-mark unhold libcap2 && clean-install ca-certificates cifs-utils util-linux e2fsprogs mount udev xfsprogs nfs-common netbase +RUN apt update && apt upgrade -y && apt-mark unhold libcap2 && clean-install ca-certificates cifs-utils util-linux e2fsprogs mount udev xfsprogs nfs-common netbase curl python3-requests COPY ./pkg/azurefile-proxy/*.sh /azurefile-proxy/ COPY ./pkg/azurefile-proxy/azurefile-proxy.service /azurefile-proxy/ COPY ./_output/${ARCH}/azurefile-proxy /azurefile-proxy/ +COPY --from=builder --chown=root:root /tmp/azfilesauth_amd64.deb /azurefile-proxy/azfilesauth_amd64.deb + RUN chmod +x /azurefile-proxy/*.sh && \ chmod +x /azurefile-proxy/azurefile-proxy.service && \ chmod +x /azurefile-proxy/azurefile-proxy +RUN if [ "$ARCH" = "amd64" ] ; then \ + clean-install libcurl4-gnutls-dev \ + && apt update && apt install -y /azurefile-proxy/azfilesauth_amd64.deb; fi + LABEL maintainers="andyzhangx" LABEL description="AzureFile CSI Driver"