Skip to content

Commit 99b17e9

Browse files
andyzhangxandyzhangx
committed
feat: support SMB mount with managed identity
chore: add kubelet mount path chore: mkdir kubeberos dir fix fix install sudo curl revert fix mountOptions support clientID support clientID in sc fix sh script golint error fix fix config file creation on host add SETUP_MI_AUTH env var fix azfilesauth_1.0 package add azfilesrefresh sidecar feat: mount with managed identity auth fix
1 parent 762805c commit 99b17e9

File tree

10 files changed

+141
-22
lines changed

10 files changed

+141
-22
lines changed

‎charts/latest/azurefile-csi-driver-v0.0.0.tgz

142 Bytes
Binary file not shown.

‎charts/latest/azurefile-csi-driver/templates/csi-azurefile-node.yaml

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,11 +96,16 @@ spec:
9696
value: "{{ .Values.linux.kubelet }}"
9797
- name: MIGRATE_K8S_REPO
9898
value: "{{ .Values.node.azurefileProxy.migrateK8sRepo }}"
99+
- name: SETUP_MI_AUTH
100+
value: "{{ .Values.node.azurefileProxy.setUpMIAuth }}"
99101
volumeMounts:
100102
- name: host-usr
101103
mountPath: /host/usr
102104
- name: host-etc
103105
mountPath: /host/etc
106+
- name: mountpoint-dir
107+
mountPath: /var/lib/kubelet/
108+
mountPropagation: Bidirectional
104109
containers:
105110
- name: liveness-probe
106111
volumeMounts:
@@ -140,6 +145,27 @@ spec:
140145
- name: registration-dir
141146
mountPath: /registration
142147
resources: {{- toYaml .Values.linux.resources.nodeDriverRegistrar | nindent 12 }}
148+
- name: azfilesrefresh
149+
{{- if hasPrefix "/" .Values.image.azurefile.repository }}
150+
image: "{{ .Values.image.baseRepo }}{{ .Values.image.azurefile.repository }}:{{ .Values.image.azurefile.tag }}"
151+
{{- else }}
152+
image: "{{ .Values.image.azurefile.repository }}:{{ .Values.image.azurefile.tag }}"
153+
{{- end }}
154+
imagePullPolicy: {{ .Values.image.azurefile.pullPolicy }}
155+
command:
156+
- "azfilesrefresh"
157+
securityContext:
158+
privileged: true
159+
capabilities:
160+
drop:
161+
- ALL
162+
volumeMounts:
163+
- mountPath: /var/lib/kubelet/
164+
mountPropagation: Bidirectional
165+
name: mountpoint-dir
166+
- name: host-etc
167+
mountPath: /etc
168+
resources: {{- toYaml .Values.linux.resources.azfilesrefresh | nindent 12 }}
143169
- name: azurefile
144170
{{- if hasPrefix "/" .Values.image.azurefile.repository }}
145171
image: "{{ .Values.image.baseRepo }}{{ .Values.image.azurefile.repository }}:{{ .Values.image.azurefile.tag }}"

‎charts/latest/azurefile-csi-driver/values.yaml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,13 @@ controller:
8888
requests:
8989
cpu: 10m
9090
memory: 20Mi
91+
azfilesrefresh:
92+
limits:
93+
cpu: 1
94+
memory: 100Mi
95+
requests:
96+
cpu: 10m
97+
memory: 20Mi
9198
azurefile:
9299
limits:
93100
cpu: 2
@@ -128,6 +135,7 @@ node:
128135
enabled: true
129136
installAznfsMount: true
130137
migrateK8sRepo: false
138+
setUpMIAuth: true
131139

132140
snapshot:
133141
enabled: false

‎deploy/csi-azurefile-node.yaml

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,11 +58,16 @@ spec:
5858
value: "true"
5959
- name: INSTALL_AZNFS_MOUNT
6060
value: "true"
61+
- name: SETUP_MI_AUTH
62+
value: "true"
6163
volumeMounts:
6264
- name: host-usr
6365
mountPath: /host/usr
6466
- name: host-etc
6567
mountPath: /host/etc
68+
- name: mountpoint-dir
69+
mountPath: /var/lib/kubelet/
70+
mountPropagation: Bidirectional
6671
containers:
6772
- name: liveness-probe
6873
volumeMounts:
@@ -163,12 +168,36 @@ spec:
163168
name: device-dir
164169
- mountPath: /run/kata-containers/shared/direct-volumes
165170
name: kata-direct-volumes
171+
- name: host-etc
172+
mountPath: /etc
166173
resources:
167174
limits:
168175
memory: 400Mi
169176
requests:
170177
cpu: 10m
171178
memory: 20Mi
179+
- name: azfilesrefresh
180+
image: mcr.microsoft.com/k8s/csi/azurefile-csi:latest
181+
imagePullPolicy: IfNotPresent
182+
command:
183+
- "azfilesrefresh"
184+
securityContext:
185+
privileged: true
186+
capabilities:
187+
drop:
188+
- ALL
189+
volumeMounts:
190+
- mountPath: /var/lib/kubelet/
191+
mountPropagation: Bidirectional
192+
name: mountpoint-dir
193+
- name: host-etc
194+
mountPath: /etc
195+
resources:
196+
limits:
197+
memory: 100Mi
198+
requests:
199+
cpu: 10m
200+
memory: 20Mi
172201
volumes:
173202
- name: host-usr
174203
hostPath:

‎pkg/azurefile-proxy/init.sh

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,16 +17,28 @@
1717
set -xe
1818

1919
MIGRATE_K8S_REPO=${MIGRATE_K8S_REPO:-false}
20+
SETUP_MI_AUTH=${SETUP_MI_AUTH:-true}
2021

2122
KUBELET_PATH=${KUBELET_PATH:-/var/lib/kubelet}
2223
if [ "$KUBELET_PATH" != "/var/lib/kubelet" ];then
2324
echo "kubelet path is $KUBELET_PATH, update azurefile-proxy.service...."
2425
sed -i "s#--azurefile-proxy-endpoint[^ ]*#--azurefile-proxy-endpoint=unix:/${KUBELET_PATH}/plugins/file.csi.azure.com/azurefile-proxy.sock#" /azurefile-proxy/azurefile-proxy.service
2526
echo "azurefile-proxy endpoint is updated to unix:/$KUBELET_PATH/plugins/file.csi.azure.com/azurefile-proxy.sock"
26-
fi
27+
fi
2728

2829
HOST_CMD="nsenter --mount=/proc/1/ns/mnt"
2930

31+
if [ "$SETUP_MI_AUTH" = "true" ];then
32+
echo "set up /etc/krb5.conf on host"
33+
printf '[libdefaults]\ndefault_ccache_name = FILE:/var/lib/kubelet/kerberos/krb5cc_%s\n' "%{uid}" > /host/etc/krb5.conf
34+
35+
mkdir -p /var/lib/kubelet/kerberos
36+
37+
echo "set up /etc/azfilesauth/config.yaml on host"
38+
mkdir -p /host/etc/azfilesauth
39+
printf 'USER_UID: 0\nKRB5_CC_NAME: /var/lib/kubelet/kerberos/krb5cc_0\n' > /host/etc/azfilesauth/config.yaml
40+
fi
41+
3042
DISTRIBUTION=$($HOST_CMD cat /etc/os-release | grep ^ID= | cut -d'=' -f2 | tr -d '"')
3143
ARCH=$($HOST_CMD uname -m)
3244
echo "Linux distribution: $DISTRIBUTION, Arch: $ARCH"

‎pkg/azurefile/azurefile.go

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -166,6 +166,7 @@ const (
166166
defaultConfidentialContainerLabel = "kubernetes.azure.com/kata-cc-isolation"
167167
runtimeClassHandlerField = "runtimeclasshandler"
168168
defaultRuntimeClassHandler = "kata-cc"
169+
mountWithManagedIdentityField = "mountwithmanagedidentity"
169170

170171
accountNotProvisioned = "StorageAccountIsNotProvisioned"
171172
// this is a workaround fix for 429 throttling issue, will update cloud provider for better fix later
@@ -782,7 +783,7 @@ func (d *Driver) GetAccountInfo(ctx context.Context, volumeID string, secrets, r
782783

783784
var protocol, accountKey, secretName, pvcNamespace string
784785
// getAccountKeyFromSecret indicates whether get account key only from k8s secret
785-
var getAccountKeyFromSecret, getLatestAccountKey bool
786+
var getAccountKeyFromSecret, getLatestAccountKey, mountWithManagedIdentity bool
786787
var clientID, tenantID, serviceAccountToken string
787788

788789
for k, v := range reqContext {
@@ -815,6 +816,10 @@ func (d *Driver) GetAccountInfo(ctx context.Context, volumeID string, secrets, r
815816
}
816817
case clientIDField:
817818
clientID = v
819+
case mountWithManagedIdentityField:
820+
if mountWithManagedIdentity, err = strconv.ParseBool(v); err != nil {
821+
return rgName, accountName, accountKey, fileShareName, diskName, subsID, fmt.Errorf("invalid %s: %s in volume context", mountWithManagedIdentityField, v)
822+
}
818823
case tenantIDField:
819824
tenantID = v
820825
case strings.ToLower(serviceAccountTokenField):
@@ -844,7 +849,11 @@ func (d *Driver) GetAccountInfo(ctx context.Context, volumeID string, secrets, r
844849
}
845850
}
846851

847-
// if client id is specified, we only use service account token to get account key
852+
if mountWithManagedIdentity {
853+
klog.V(2).Infof("mountWithManagedIdentity is true, use managed identity auth")
854+
return rgName, accountName, accountKey, fileShareName, diskName, subsID, nil
855+
}
856+
848857
if clientID != "" {
849858
klog.V(2).Infof("clientID(%s) is specified, use service account token to get account key", clientID)
850859
accountKey, err := d.cloud.GetStorageAccesskeyFromServiceAccountToken(ctx, subsID, accountName, rgName, clientID, tenantID, serviceAccountToken)

‎pkg/azurefile/controllerserver.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -227,6 +227,7 @@ func (d *Driver) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest)
227227
fileShareNameReplaceMap[pvNameMetadata] = v
228228
case serverNameField:
229229
case folderNameField:
230+
case mountWithManagedIdentityField:
230231
case clientIDField:
231232
case confidentialContainerLabelField:
232233
case runtimeClassHandlerField:

‎pkg/azurefile/nodeserver.go

Lines changed: 37 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -76,8 +76,7 @@ func (d *Driver) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolu
7676
mountPermissions := d.mountPermissions
7777
context := req.GetVolumeContext()
7878
if context != nil {
79-
// token request
80-
if getValueInMap(context, serviceAccountTokenField) != "" && getValueInMap(context, clientIDField) != "" {
79+
if !strings.EqualFold(getValueInMap(context, mountWithManagedIdentityField), trueValue) && getValueInMap(context, serviceAccountTokenField) != "" && getValueInMap(context, clientIDField) != "" {
8180
klog.V(2).Infof("NodePublishVolume: volume(%s) mount on %s with service account token, clientID: %s", volumeID, target, getValueInMap(context, clientIDField))
8281
_, err := d.NodeStageVolume(ctx, &csi.NodeStageVolumeRequest{
8382
StagingTargetPath: target,
@@ -248,7 +247,7 @@ func (d *Driver) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRe
248247
volumeID := req.GetVolumeId()
249248
context := req.GetVolumeContext()
250249

251-
if getValueInMap(context, clientIDField) != "" && getValueInMap(context, serviceAccountTokenField) == "" {
250+
if getValueInMap(context, clientIDField) != "" && !strings.EqualFold(getValueInMap(context, mountWithManagedIdentityField), trueValue) && getValueInMap(context, serviceAccountTokenField) == "" {
252251
klog.V(2).Infof("Skip NodeStageVolume for volume(%s) since clientID %s is provided but service account token is empty", volumeID, getValueInMap(context, clientIDField))
253252
return &csi.NodeStageVolumeResponse{}, nil
254253
}
@@ -277,9 +276,8 @@ func (d *Driver) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRe
277276
}
278277
// don't respect fsType from req.GetVolumeCapability().GetMount().GetFsType()
279278
// since it's ext4 by default on Linux
280-
var fsType, server, protocol, ephemeralVolMountOptions, storageEndpointSuffix, folderName string
281-
var ephemeralVol bool
282-
var encryptInTransit bool
279+
var fsType, server, protocol, ephemeralVolMountOptions, storageEndpointSuffix, folderName, clientID string
280+
var ephemeralVol, encryptInTransit, mountWithManagedIdentity bool
283281
fileShareNameReplaceMap := map[string]string{}
284282

285283
mountPermissions := d.mountPermissions
@@ -313,7 +311,6 @@ func (d *Driver) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRe
313311
fileShareNameReplaceMap[pvNameMetadata] = v
314312
case mountPermissionsField:
315313
if v != "" {
316-
var err error
317314
var perm uint64
318315
if perm, err = strconv.ParseUint(v, 8, 32); err != nil {
319316
return nil, status.Errorf(codes.InvalidArgument, "invalid mountPermissions %s", v)
@@ -325,11 +322,17 @@ func (d *Driver) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRe
325322
}
326323
}
327324
case encryptInTransitField:
328-
var err error
329325
encryptInTransit, err = strconv.ParseBool(v)
330326
if err != nil {
331327
return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("Volume context property %q must be a boolean value: %v", k, err))
332328
}
329+
case mountWithManagedIdentityField:
330+
mountWithManagedIdentity, err = strconv.ParseBool(v)
331+
if err != nil {
332+
return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("Volume context property %q must be a boolean value: %v", k, err))
333+
}
334+
case clientIDField:
335+
clientID = v
333336
}
334337
}
335338

@@ -394,18 +397,29 @@ func (d *Driver) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRe
394397
mountOptions = util.JoinMountOptions(mountFlags, []string{"vers=4,minorversion=1,sec=sys"})
395398
mountOptions = appendDefaultNfsMountOptions(mountOptions, d.appendNoResvPortOption, d.appendActimeoOption)
396399
} else {
397-
if accountName == "" || accountKey == "" {
398-
return nil, status.Errorf(codes.Internal, "accountName(%s) or accountKey is empty", accountName)
399-
}
400-
if runtime.GOOS == "windows" {
401-
mountOptions = []string{fmt.Sprintf("AZURE\\%s", accountName)}
402-
sensitiveMountOptions = []string{accountKey}
400+
if mountWithManagedIdentity && runtime.GOOS != "windows" {
401+
if clientID == "" {
402+
clientID = d.cloud.Config.AzureAuthConfig.UserAssignedIdentityID
403+
}
404+
sensitiveMountOptions = []string{"sec=krb5,cruid=0,upcall_target=mount", fmt.Sprintf("username=%s", clientID)}
405+
klog.V(2).Infof("using managed identity %s for volume %s with mount options: %v", clientID, volumeID, sensitiveMountOptions)
403406
} else {
404-
if err := os.MkdirAll(targetPath, os.FileMode(mountPermissions)); err != nil {
405-
return nil, status.Error(codes.Internal, fmt.Sprintf("MkdirAll %s failed with error: %v", targetPath, err))
407+
if accountName == "" || accountKey == "" {
408+
return nil, status.Errorf(codes.Internal, "accountName(%s) or accountKey is empty", accountName)
406409
}
407-
// parameters suggested by https://azure.microsoft.com/en-us/documentation/articles/storage-how-to-use-files-linux/
408-
sensitiveMountOptions = []string{fmt.Sprintf("username=%s,password=%s", accountName, accountKey)}
410+
if runtime.GOOS == "windows" {
411+
mountOptions = []string{fmt.Sprintf("AZURE\\%s", accountName)}
412+
sensitiveMountOptions = []string{accountKey}
413+
} else {
414+
if err := os.MkdirAll(targetPath, os.FileMode(mountPermissions)); err != nil {
415+
return nil, status.Error(codes.Internal, fmt.Sprintf("MkdirAll %s failed with error: %v", targetPath, err))
416+
}
417+
// parameters suggested by https://azure.microsoft.com/en-us/documentation/articles/storage-how-to-use-files-linux/
418+
sensitiveMountOptions = []string{fmt.Sprintf("username=%s,password=%s", accountName, accountKey)}
419+
}
420+
}
421+
422+
if runtime.GOOS != "windows" {
409423
if ephemeralVol {
410424
cifsMountFlags = util.JoinMountOptions(cifsMountFlags, strings.Split(ephemeralVolMountOptions, ","))
411425
}
@@ -449,6 +463,11 @@ func (d *Driver) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRe
449463
klog.V(2).Infof("mount with proxy succeeded for %s", cifsMountPath)
450464
} else {
451465
execFunc := func() error {
466+
if mountWithManagedIdentity && protocol != nfs && runtime.GOOS != "windows" {
467+
if out, err := setCredentialCache(server, clientID); err != nil {
468+
return fmt.Errorf("setCredentialCache failed for %s with error: %v, output: %s", server, err, out)
469+
}
470+
}
452471
return SMBMount(d.mounter, source, cifsMountPath, mountFsType, mountOptions, sensitiveMountOptions)
453472
}
454473
timeoutFunc := func() error { return fmt.Errorf("time out") }

‎pkg/azurefile/utils.go

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
"context"
2121
"fmt"
2222
"os"
23+
"os/exec"
2324
"regexp"
2425
"strconv"
2526
"strings"
@@ -372,3 +373,10 @@ func removeOptionIfExists(options []string, removeOption string) ([]string, bool
372373
}
373374
return options, false
374375
}
376+
377+
func setCredentialCache(server, clientID string) ([]byte, error) {
378+
cmd := exec.Command("azfilesauthmanager", "set", "https://"+server, "--imds-client-id", clientID)
379+
cmd.Env = append(os.Environ(), cmd.Env...)
380+
klog.V(2).Infof("Executing command: %q", cmd.String())
381+
return cmd.CombinedOutput()
382+
}

‎pkg/azurefileplugin/Dockerfile

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ ARG ARCH
2323
RUN apt update \
2424
&& apt install -y curl \
2525
&& curl -Ls https://github.com/Azure/azure-storage-azcopy/releases/download/v10.30.0/azcopy_linux_${ARCH}_10.30.0.tar.gz \
26+
&& curl -Lso /tmp/azfilesauth_amd64.deb https://raw.githubusercontent.com/andyzhangx/demo/refs/heads/master/aks/azfilesauth_1.0-4_${ARCH}.deb \
2627
| tar xvzf - --strip-components=1 -C /usr/local/bin/ --wildcards "*/azcopy"
2728

2829
FROM base
@@ -33,16 +34,22 @@ ARG binary=./_output/${ARCH}/azurefileplugin
3334
COPY ${binary} /azurefileplugin
3435
COPY --from=builder --chown=root:root /usr/local/bin/azcopy /usr/local/bin/azcopy
3536

36-
RUN apt update && apt upgrade -y && apt-mark unhold libcap2 && clean-install ca-certificates cifs-utils util-linux e2fsprogs mount udev xfsprogs nfs-common netbase
37+
RUN apt update && apt upgrade -y && apt-mark unhold libcap2 && clean-install ca-certificates cifs-utils util-linux e2fsprogs mount udev xfsprogs nfs-common netbase curl python3-requests
3738

3839
COPY ./pkg/azurefile-proxy/*.sh /azurefile-proxy/
3940
COPY ./pkg/azurefile-proxy/azurefile-proxy.service /azurefile-proxy/
4041
COPY ./_output/${ARCH}/azurefile-proxy /azurefile-proxy/
4142

43+
COPY --from=builder --chown=root:root /tmp/azfilesauth_amd64.deb /azurefile-proxy/azfilesauth_amd64.deb
44+
4245
RUN chmod +x /azurefile-proxy/*.sh && \
4346
chmod +x /azurefile-proxy/azurefile-proxy.service && \
4447
chmod +x /azurefile-proxy/azurefile-proxy
4548

49+
RUN if [ "$ARCH" = "amd64" ] ; then \
50+
clean-install libcurl4-gnutls-dev \
51+
&& apt update && apt install -y /azurefile-proxy/azfilesauth_amd64.deb; fi
52+
4653
LABEL maintainers="andyzhangx"
4754
LABEL description="AzureFile CSI Driver"
4855

0 commit comments

Comments
 (0)