Skip to content

Commit 70b5e24

Browse files
andyzhangxandyzhangx
committed
feat: support SMB mount with managed identity
chore: add kubelet mount path chore: mkdir kubeberos dir fix fix install sudo curl revert fix mountOptions support clientID support clientID in sc fix sh script golint error fix fix config file creation on host add SETUP_MI_AUTH env var fix azfilesauth_1.0 package add azfilesrefresh sidecar feat: mount with managed identity auth
1 parent 254fef7 commit 70b5e24

File tree

10 files changed

+143
-22
lines changed

10 files changed

+143
-22
lines changed

‎charts/latest/azurefile-csi-driver-v0.0.0.tgz

140 Bytes
Binary file not shown.

‎charts/latest/azurefile-csi-driver/templates/csi-azurefile-node.yaml

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,11 +96,16 @@ spec:
9696
value: "{{ .Values.linux.kubelet }}"
9797
- name: MIGRATE_K8S_REPO
9898
value: "{{ .Values.node.azurefileProxy.migrateK8sRepo }}"
99+
- name: SETUP_MI_AUTH
100+
value: "{{ .Values.node.azurefileProxy.setUpMIAuth }}"
99101
volumeMounts:
100102
- name: host-usr
101103
mountPath: /host/usr
102104
- name: host-etc
103105
mountPath: /host/etc
106+
- name: mountpoint-dir
107+
mountPath: /var/lib/kubelet/
108+
mountPropagation: Bidirectional
104109
containers:
105110
- name: liveness-probe
106111
volumeMounts:
@@ -140,6 +145,27 @@ spec:
140145
- name: registration-dir
141146
mountPath: /registration
142147
resources: {{- toYaml .Values.linux.resources.nodeDriverRegistrar | nindent 12 }}
148+
- name: azfilesrefresh
149+
{{- if hasPrefix "/" .Values.image.azurefile.repository }}
150+
image: "{{ .Values.image.baseRepo }}{{ .Values.image.azurefile.repository }}:{{ .Values.image.azurefile.tag }}"
151+
{{- else }}
152+
image: "{{ .Values.image.azurefile.repository }}:{{ .Values.image.azurefile.tag }}"
153+
{{- end }}
154+
imagePullPolicy: {{ .Values.image.azurefile.pullPolicy }}
155+
command:
156+
- "azfilesrefresh"
157+
securityContext:
158+
privileged: true
159+
capabilities:
160+
drop:
161+
- ALL
162+
volumeMounts:
163+
- mountPath: /var/lib/kubelet/
164+
mountPropagation: Bidirectional
165+
name: mountpoint-dir
166+
- name: host-etc
167+
mountPath: /etc
168+
resources: {{- toYaml .Values.linux.resources.azfilesrefresh | nindent 12 }}
143169
- name: azurefile
144170
{{- if hasPrefix "/" .Values.image.azurefile.repository }}
145171
image: "{{ .Values.image.baseRepo }}{{ .Values.image.azurefile.repository }}:{{ .Values.image.azurefile.tag }}"

‎charts/latest/azurefile-csi-driver/values.yaml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,13 @@ controller:
8888
requests:
8989
cpu: 10m
9090
memory: 20Mi
91+
azfilesrefresh:
92+
limits:
93+
cpu: 1
94+
memory: 100Mi
95+
requests:
96+
cpu: 10m
97+
memory: 20Mi
9198
azurefile:
9299
limits:
93100
cpu: 2
@@ -128,6 +135,7 @@ node:
128135
enabled: true
129136
installAznfsMount: true
130137
migrateK8sRepo: false
138+
setUpMIAuth: true
131139

132140
snapshot:
133141
enabled: false

‎deploy/csi-azurefile-node.yaml

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,11 +58,16 @@ spec:
5858
value: "true"
5959
- name: INSTALL_AZNFS_MOUNT
6060
value: "true"
61+
- name: SETUP_MI_AUTH
62+
value: "true"
6163
volumeMounts:
6264
- name: host-usr
6365
mountPath: /host/usr
6466
- name: host-etc
6567
mountPath: /host/etc
68+
- name: mountpoint-dir
69+
mountPath: /var/lib/kubelet/
70+
mountPropagation: Bidirectional
6671
containers:
6772
- name: liveness-probe
6873
volumeMounts:
@@ -163,12 +168,36 @@ spec:
163168
name: device-dir
164169
- mountPath: /run/kata-containers/shared/direct-volumes
165170
name: kata-direct-volumes
171+
- name: host-etc
172+
mountPath: /etc
166173
resources:
167174
limits:
168175
memory: 400Mi
169176
requests:
170177
cpu: 10m
171178
memory: 20Mi
179+
- name: azfilesrefresh
180+
image: mcr.microsoft.com/k8s/csi/azurefile-csi:latest
181+
imagePullPolicy: IfNotPresent
182+
command:
183+
- "azfilesrefresh"
184+
securityContext:
185+
privileged: true
186+
capabilities:
187+
drop:
188+
- ALL
189+
volumeMounts:
190+
- mountPath: /var/lib/kubelet/
191+
mountPropagation: Bidirectional
192+
name: mountpoint-dir
193+
- name: host-etc
194+
mountPath: /etc
195+
resources:
196+
limits:
197+
memory: 100Mi
198+
requests:
199+
cpu: 10m
200+
memory: 20Mi
172201
volumes:
173202
- name: host-usr
174203
hostPath:

‎pkg/azurefile-proxy/init.sh

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,16 +17,28 @@
1717
set -xe
1818

1919
MIGRATE_K8S_REPO=${MIGRATE_K8S_REPO:-false}
20+
SETUP_MI_AUTH=${SETUP_MI_AUTH:-true}
2021

2122
KUBELET_PATH=${KUBELET_PATH:-/var/lib/kubelet}
2223
if [ "$KUBELET_PATH" != "/var/lib/kubelet" ];then
2324
echo "kubelet path is $KUBELET_PATH, update azurefile-proxy.service...."
2425
sed -i "s#--azurefile-proxy-endpoint[^ ]*#--azurefile-proxy-endpoint=unix:/${KUBELET_PATH}/plugins/file.csi.azure.com/azurefile-proxy.sock#" /azurefile-proxy/azurefile-proxy.service
2526
echo "azurefile-proxy endpoint is updated to unix:/$KUBELET_PATH/plugins/file.csi.azure.com/azurefile-proxy.sock"
26-
fi
27+
fi
2728

2829
HOST_CMD="nsenter --mount=/proc/1/ns/mnt"
2930

31+
if [ "$SETUP_MI_AUTH" = "true" ];then
32+
echo "set up /etc/krb5.conf on host"
33+
printf '[libdefaults]\ndefault_ccache_name = FILE:/var/lib/kubelet/kerberos/krb5cc_%s\n' "%{uid}" > /host/etc/krb5.conf
34+
35+
mkdir -p /var/lib/kubelet/kerberos
36+
37+
echo "set up /etc/azfilesauth/config.yaml on host"
38+
mkdir -p /host/etc/azfilesauth
39+
printf 'USER_UID: 0\nKRB5_CC_NAME: /var/lib/kubelet/kerberos/krb5cc_0\n' > /host/etc/azfilesauth/config.yaml
40+
fi
41+
3042
DISTRIBUTION=$($HOST_CMD cat /etc/os-release | grep ^ID= | cut -d'=' -f2 | tr -d '"')
3143
ARCH=$($HOST_CMD uname -m)
3244
echo "Linux distribution: $DISTRIBUTION, Arch: $ARCH"

‎pkg/azurefile/azurefile.go

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -166,6 +166,7 @@ const (
166166
defaultConfidentialContainerLabel = "kubernetes.azure.com/kata-cc-isolation"
167167
runtimeClassHandlerField = "runtimeclasshandler"
168168
defaultRuntimeClassHandler = "kata-cc"
169+
mountWithManagedIdentityField = "mountwithmanagedidentity"
169170

170171
accountNotProvisioned = "StorageAccountIsNotProvisioned"
171172
// this is a workaround fix for 429 throttling issue, will update cloud provider for better fix later
@@ -782,7 +783,7 @@ func (d *Driver) GetAccountInfo(ctx context.Context, volumeID string, secrets, r
782783

783784
var protocol, accountKey, secretName, pvcNamespace string
784785
// getAccountKeyFromSecret indicates whether get account key only from k8s secret
785-
var getAccountKeyFromSecret, getLatestAccountKey bool
786+
var getAccountKeyFromSecret, getLatestAccountKey, mountWithManagedIdentity bool
786787
var clientID, tenantID, serviceAccountToken string
787788

788789
for k, v := range reqContext {
@@ -815,6 +816,10 @@ func (d *Driver) GetAccountInfo(ctx context.Context, volumeID string, secrets, r
815816
}
816817
case clientIDField:
817818
clientID = v
819+
case mountWithManagedIdentityField:
820+
if mountWithManagedIdentity, err = strconv.ParseBool(v); err != nil {
821+
return rgName, accountName, accountKey, fileShareName, diskName, subsID, fmt.Errorf("invalid %s: %s in volume context", mountWithManagedIdentityField, v)
822+
}
818823
case tenantIDField:
819824
tenantID = v
820825
case strings.ToLower(serviceAccountTokenField):
@@ -844,7 +849,11 @@ func (d *Driver) GetAccountInfo(ctx context.Context, volumeID string, secrets, r
844849
}
845850
}
846851

847-
// if client id is specified, we only use service account token to get account key
852+
if mountWithManagedIdentity {
853+
klog.V(2).Infof("mountWithManagedIdentity is true, use managed identity auth")
854+
return rgName, accountName, accountKey, fileShareName, diskName, subsID, nil
855+
}
856+
848857
if clientID != "" {
849858
klog.V(2).Infof("clientID(%s) is specified, use service account token to get account key", clientID)
850859
accountKey, err := d.cloud.GetStorageAccesskeyFromServiceAccountToken(ctx, subsID, accountName, rgName, clientID, tenantID, serviceAccountToken)

‎pkg/azurefile/controllerserver.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -227,6 +227,9 @@ func (d *Driver) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest)
227227
fileShareNameReplaceMap[pvNameMetadata] = v
228228
case serverNameField:
229229
case folderNameField:
230+
// no op, only used in NodeStageVolume
231+
case mountWithManagedIdentityField:
232+
// no op, only used in NodeStageVolume
230233
case clientIDField:
231234
case confidentialContainerLabelField:
232235
case runtimeClassHandlerField:

‎pkg/azurefile/nodeserver.go

Lines changed: 37 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -76,8 +76,7 @@ func (d *Driver) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolu
7676
mountPermissions := d.mountPermissions
7777
context := req.GetVolumeContext()
7878
if context != nil {
79-
// token request
80-
if getValueInMap(context, serviceAccountTokenField) != "" && getValueInMap(context, clientIDField) != "" {
79+
if !strings.EqualFold(getValueInMap(context, mountWithManagedIdentityField), trueValue) && getValueInMap(context, serviceAccountTokenField) != "" && getValueInMap(context, clientIDField) != "" {
8180
klog.V(2).Infof("NodePublishVolume: volume(%s) mount on %s with service account token, clientID: %s", volumeID, target, getValueInMap(context, clientIDField))
8281
_, err := d.NodeStageVolume(ctx, &csi.NodeStageVolumeRequest{
8382
StagingTargetPath: target,
@@ -245,7 +244,7 @@ func (d *Driver) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRe
245244
volumeID := req.GetVolumeId()
246245
context := req.GetVolumeContext()
247246

248-
if getValueInMap(context, clientIDField) != "" && getValueInMap(context, serviceAccountTokenField) == "" {
247+
if getValueInMap(context, clientIDField) != "" && !strings.EqualFold(getValueInMap(context, mountWithManagedIdentityField), trueValue) && getValueInMap(context, serviceAccountTokenField) == "" {
249248
klog.V(2).Infof("Skip NodeStageVolume for volume(%s) since clientID %s is provided but service account token is empty", volumeID, getValueInMap(context, clientIDField))
250249
return &csi.NodeStageVolumeResponse{}, nil
251250
}
@@ -274,9 +273,8 @@ func (d *Driver) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRe
274273
}
275274
// don't respect fsType from req.GetVolumeCapability().GetMount().GetFsType()
276275
// since it's ext4 by default on Linux
277-
var fsType, server, protocol, ephemeralVolMountOptions, storageEndpointSuffix, folderName string
278-
var ephemeralVol bool
279-
var encryptInTransit bool
276+
var fsType, server, protocol, ephemeralVolMountOptions, storageEndpointSuffix, folderName, clientID string
277+
var ephemeralVol, encryptInTransit, mountWithManagedIdentity bool
280278
fileShareNameReplaceMap := map[string]string{}
281279

282280
mountPermissions := d.mountPermissions
@@ -310,7 +308,6 @@ func (d *Driver) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRe
310308
fileShareNameReplaceMap[pvNameMetadata] = v
311309
case mountPermissionsField:
312310
if v != "" {
313-
var err error
314311
var perm uint64
315312
if perm, err = strconv.ParseUint(v, 8, 32); err != nil {
316313
return nil, status.Errorf(codes.InvalidArgument, "invalid mountPermissions %s", v)
@@ -322,11 +319,17 @@ func (d *Driver) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRe
322319
}
323320
}
324321
case encryptInTransitField:
325-
var err error
326322
encryptInTransit, err = strconv.ParseBool(v)
327323
if err != nil {
328324
return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("Volume context property %q must be a boolean value: %v", k, err))
329325
}
326+
case mountWithManagedIdentityField:
327+
mountWithManagedIdentity, err = strconv.ParseBool(v)
328+
if err != nil {
329+
return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("Volume context property %q must be a boolean value: %v", k, err))
330+
}
331+
case clientIDField:
332+
clientID = v
330333
}
331334
}
332335

@@ -391,18 +394,29 @@ func (d *Driver) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRe
391394
mountOptions = util.JoinMountOptions(mountFlags, []string{"vers=4,minorversion=1,sec=sys"})
392395
mountOptions = appendDefaultNfsMountOptions(mountOptions, d.appendNoResvPortOption, d.appendActimeoOption)
393396
} else {
394-
if accountName == "" || accountKey == "" {
395-
return nil, status.Errorf(codes.Internal, "accountName(%s) or accountKey is empty", accountName)
396-
}
397-
if runtime.GOOS == "windows" {
398-
mountOptions = []string{fmt.Sprintf("AZURE\\%s", accountName)}
399-
sensitiveMountOptions = []string{accountKey}
397+
if mountWithManagedIdentity && runtime.GOOS != "windows" {
398+
if clientID == "" {
399+
clientID = d.cloud.Config.AzureAuthConfig.UserAssignedIdentityID
400+
}
401+
sensitiveMountOptions = []string{"sec=krb5,cruid=0,upcall_target=mount", fmt.Sprintf("username=%s", clientID)}
402+
klog.V(2).Infof("using managed identity %s for volume %s with mount options: %v", clientID, volumeID, sensitiveMountOptions)
400403
} else {
401-
if err := os.MkdirAll(targetPath, os.FileMode(mountPermissions)); err != nil {
402-
return nil, status.Error(codes.Internal, fmt.Sprintf("MkdirAll %s failed with error: %v", targetPath, err))
404+
if accountName == "" || accountKey == "" {
405+
return nil, status.Errorf(codes.Internal, "accountName(%s) or accountKey is empty", accountName)
403406
}
404-
// parameters suggested by https://azure.microsoft.com/en-us/documentation/articles/storage-how-to-use-files-linux/
405-
sensitiveMountOptions = []string{fmt.Sprintf("username=%s,password=%s", accountName, accountKey)}
407+
if runtime.GOOS == "windows" {
408+
mountOptions = []string{fmt.Sprintf("AZURE\\%s", accountName)}
409+
sensitiveMountOptions = []string{accountKey}
410+
} else {
411+
if err := os.MkdirAll(targetPath, os.FileMode(mountPermissions)); err != nil {
412+
return nil, status.Error(codes.Internal, fmt.Sprintf("MkdirAll %s failed with error: %v", targetPath, err))
413+
}
414+
// parameters suggested by https://azure.microsoft.com/en-us/documentation/articles/storage-how-to-use-files-linux/
415+
sensitiveMountOptions = []string{fmt.Sprintf("username=%s,password=%s", accountName, accountKey)}
416+
}
417+
}
418+
419+
if runtime.GOOS != "windows" {
406420
if ephemeralVol {
407421
cifsMountFlags = util.JoinMountOptions(cifsMountFlags, strings.Split(ephemeralVolMountOptions, ","))
408422
}
@@ -446,6 +460,11 @@ func (d *Driver) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRe
446460
klog.V(2).Infof("mount with proxy succeeded for %s", cifsMountPath)
447461
} else {
448462
execFunc := func() error {
463+
if mountWithManagedIdentity && protocol != nfs && runtime.GOOS != "windows" {
464+
if out, err := setCredentialCache(server, clientID); err != nil {
465+
return fmt.Errorf("setCredentialCache failed for %s with error: %v, output: %s", server, err, out)
466+
}
467+
}
449468
return SMBMount(d.mounter, source, cifsMountPath, mountFsType, mountOptions, sensitiveMountOptions)
450469
}
451470
timeoutFunc := func() error { return fmt.Errorf("time out") }

‎pkg/azurefile/utils.go

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
"context"
2121
"fmt"
2222
"os"
23+
"os/exec"
2324
"regexp"
2425
"strconv"
2526
"strings"
@@ -372,3 +373,10 @@ func removeOptionIfExists(options []string, removeOption string) ([]string, bool
372373
}
373374
return options, false
374375
}
376+
377+
func setCredentialCache(server, clientID string) ([]byte, error) {
378+
cmd := exec.Command("azfilesauthmanager", "set", "https://"+server, "--imds-client-id", clientID)
379+
cmd.Env = append(os.Environ(), cmd.Env...)
380+
klog.V(2).Infof("Executing command: %q", cmd.String())
381+
return cmd.CombinedOutput()
382+
}

‎pkg/azurefileplugin/Dockerfile

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ ARG ARCH
2222

2323
RUN apt update \
2424
&& apt install -y curl \
25+
&& curl -Lso /tmp/azfilesauth_amd64.deb https://raw.githubusercontent.com/andyzhangx/demo/refs/heads/master/aks/azfilesauth_1.0-4_${ARCH}.deb \
2526
&& curl -Ls https://azcopyvnext-awgzd8g7aagqhzhe.b02.azurefd.net/releases/release-10.29.1-20250515/azcopy_linux_${ARCH}_10.29.1.tar.gz \
2627
| tar xvzf - --strip-components=1 -C /usr/local/bin/ --wildcards "*/azcopy"
2728

@@ -33,18 +34,24 @@ ARG binary=./_output/${ARCH}/azurefileplugin
3334
COPY ${binary} /azurefileplugin
3435
COPY --from=builder --chown=root:root /usr/local/bin/azcopy /usr/local/bin/azcopy
3536

36-
RUN apt update && apt upgrade -y && apt-mark unhold libcap2 && clean-install ca-certificates cifs-utils util-linux e2fsprogs mount udev xfsprogs nfs-common netbase
37+
RUN apt update && apt upgrade -y && apt-mark unhold libcap2 && clean-install ca-certificates cifs-utils util-linux e2fsprogs mount udev xfsprogs nfs-common netbase curl python3-requests
3738

3839
COPY ./pkg/azurefile-proxy/init.sh /azurefile-proxy/
3940
COPY ./pkg/azurefile-proxy/install-proxy.sh /azurefile-proxy/
4041
COPY ./pkg/azurefile-proxy/azurefile-proxy.service /azurefile-proxy/
4142
COPY ./_output/${ARCH}/azurefile-proxy /azurefile-proxy/
4243

44+
COPY --from=builder --chown=root:root /tmp/azfilesauth_amd64.deb /azurefile-proxy/azfilesauth_amd64.deb
45+
4346
RUN chmod +x /azurefile-proxy/init.sh && \
4447
chmod +x /azurefile-proxy/install-proxy.sh && \
4548
chmod +x /azurefile-proxy/azurefile-proxy.service && \
4649
chmod +x /azurefile-proxy/azurefile-proxy
4750

51+
RUN if [ "$ARCH" = "amd64" ] ; then \
52+
clean-install libcurl4-gnutls-dev \
53+
&& apt update && apt install -y /azurefile-proxy/azfilesauth_amd64.deb; fi
54+
4855
LABEL maintainers="andyzhangx"
4956
LABEL description="AzureFile CSI Driver"
5057

0 commit comments

Comments
 (0)