Skip to content

Commit 78b9a9e

Browse files
cvvzcvvz
committed
feat: support workload identity setting in static PV mount (#1204)
* feat: support workload identity setting in static PV mount * fix
1 parent cf3db2e commit 78b9a9e

File tree

5 files changed

+225
-0
lines changed

5 files changed

+225
-0
lines changed

‎charts/latest/blob-csi-driver/templates/csi-blob-driver.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,3 +12,5 @@ spec:
1212
volumeLifecycleModes:
1313
- Persistent
1414
- Ephemeral
15+
tokenRequests:
16+
- audience: api://AzureADTokenExchange

‎docs/workload-identity.md renamed to ‎docs/workload-identity-deploy-csi-driver.md

File renamed without changes.

‎docs/workload-identity-static-pv-mount.md

Lines changed: 178 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,178 @@
1+
# Example of static PV mount with workload identity
2+
3+
> Note:
4+
> - Available kubernetes version >= v1.20
5+
6+
## prerequisite
7+
8+
9+
### 1. Create a cluster with oidc-issuer enabled and get the credential
10+
11+
Following the [documentation](https://learn.microsoft.com/en-us/azure/aks/use-oidc-issuer#create-an-aks-cluster-with-oidc-issuer) to create an AKS cluster with the `--enable-oidc-issuer` parameter and get the AKS credentials. And export following environment variables:
12+
```
13+
export RESOURCE_GROUP=<your resource group name>
14+
export CLUSTER_NAME=<your cluster name>
15+
export REGION=<your region>
16+
```
17+
18+
19+
### 2. Create a new storage account and container
20+
21+
Following the [documentation](https://learn.microsoft.com/en-us/azure/storage/blobs/storage-quickstart-blobs-cli) to create a new storage account and container or use your own. And export following environment variables:
22+
```
23+
export STORAGE_RESOURCE_GROUP=<your storage account resource group>
24+
export ACCOUNT=<your storage account name>
25+
export CONTAINER=<your container name>
26+
```
27+
28+
### 3. Create managed identity and role assignment
29+
```
30+
export UAMI=<your managed identity name>
31+
az identity create --name $UAMI --resource-group $RESOURCE_GROUP
32+
33+
export USER_ASSIGNED_CLIENT_ID="$(az identity show -g $RESOURCE_GROUP --name $UAMI --query 'clientId' -o tsv)"
34+
export IDENTITY_TENANT=$(az aks show --name $CLUSTER_NAME --resource-group $RESOURCE_GROUP --query identity.tenantId -o tsv)
35+
export ACCOUNT_SCOPE=$(az storage account show --name $ACCOUNT --query id -o tsv)
36+
37+
# please retry if you meet `Cannot find user or service principal in graph database` error, it may take a while for the identity to propagate
38+
az role assignment create --role "Storage Account Contributor" --assignee $USER_ASSIGNED_CLIENT_ID --scope $ACCOUNT_SCOPE
39+
```
40+
41+
### 4. Create service account on AKS
42+
```
43+
export SERVICE_ACCOUNT_NAME=<your sa name>
44+
export SERVICE_ACCOUNT_NAMESPACE=<your sa namespace>
45+
46+
cat <<EOF | kubectl apply -f -
47+
apiVersion: v1
48+
kind: ServiceAccount
49+
metadata:
50+
name: ${SERVICE_ACCOUNT_NAME}
51+
namespace: ${SERVICE_ACCOUNT_NAMESPACE}
52+
EOF
53+
```
54+
55+
### 5. Create the federated identity credential between the managed identity, service account issuer, and subject using the `az identity federated-credential create` command.
56+
```
57+
export FEDERATED_IDENTITY_NAME=<your federated identity name>
58+
export AKS_OIDC_ISSUER="$(az aks show --resource-group $RESOURCE_GROUP --name $CLUSTER_NAME --query "oidcIssuerProfile.issuerUrl" -o tsv)"
59+
60+
az identity federated-credential create --name $FEDERATED_IDENTITY_NAME \
61+
--identity-name $UAMI \
62+
--resource-group $RESOURCE_GROUP \
63+
--issuer $AKS_OIDC_ISSUER \
64+
--subject system:serviceaccount:${SERVICE_ACCOUNT_NAMESPACE}:${SERVICE_ACCOUNT_NAME}
65+
```
66+
67+
## option#1: static provision with PV
68+
```
69+
cat <<EOF | kubectl apply -f -
70+
apiVersion: v1
71+
kind: PersistentVolume
72+
metadata:
73+
annotations:
74+
pv.kubernetes.io/provisioned-by: blob.csi.azure.com
75+
name: pv-blob
76+
spec:
77+
capacity:
78+
storage: 10Gi
79+
accessModes:
80+
- ReadWriteMany
81+
persistentVolumeReclaimPolicy: Retain
82+
storageClassName: blob-fuse
83+
mountOptions:
84+
- -o allow_other
85+
- --file-cache-timeout-in-seconds=120
86+
csi:
87+
driver: blob.csi.azure.com
88+
# make sure volumeid is unique for every storage blob container in the cluster
89+
# the # character is reserved for internal use, the / character is not allowed
90+
volumeHandle: unique_volume_id
91+
volumeAttributes:
92+
storageaccount: $ACCOUNT # required
93+
containerName: $CONTAINER # required
94+
clientID: $USER_ASSIGNED_CLIENT_ID # required
95+
resourcegroup: $STORAGE_RESOURCE_GROUP # optional, specified when the storage account is not under AKS node resource group(which is prefixed with "MC_")
96+
# tenantID: $IDENTITY_TENANT #optional, only specified when workload identity and AKS cluster are in different tenant
97+
# subscriptionid: $SUBSCRIPTION #optional, only specified when workload identity and AKS cluster are in different subscription
98+
---
99+
apiVersion: apps/v1
100+
kind: StatefulSet
101+
metadata:
102+
name: statefulset-blob
103+
labels:
104+
app: nginx
105+
spec:
106+
serviceName: statefulset-blob
107+
replicas: 1
108+
template:
109+
metadata:
110+
labels:
111+
app: nginx
112+
spec:
113+
serviceAccountName: $SERVICE_ACCOUNT_NAME #required, Pod does not use this service account has no permission to mount the volume
114+
nodeSelector:
115+
"kubernetes.io/os": linux
116+
containers:
117+
- name: statefulset-blob
118+
image: mcr.microsoft.com/oss/nginx/nginx:1.19.5
119+
command:
120+
- "/bin/bash"
121+
- "-c"
122+
- set -euo pipefail; while true; do echo $(date) >> /mnt/blob/outfile; sleep 1; done
123+
volumeMounts:
124+
- name: persistent-storage
125+
mountPath: /mnt/blob
126+
readOnly: false
127+
updateStrategy:
128+
type: RollingUpdate
129+
selector:
130+
matchLabels:
131+
app: nginx
132+
volumeClaimTemplates:
133+
- metadata:
134+
name: persistent-storage
135+
spec:
136+
storageClassName: blob-fuse
137+
accessModes: ["ReadWriteMany"]
138+
resources:
139+
requests:
140+
storage: 10Gi
141+
EOF
142+
```
143+
144+
## option#2: Pod with ephemeral inline volume
145+
```
146+
cat <<EOF | kubectl apply -f -
147+
kind: Pod
148+
apiVersion: v1
149+
metadata:
150+
name: nginx-blobfuse-inline-volume
151+
spec:
152+
serviceAccountName: $SERVICE_ACCOUNT_NAME #required, Pod does not use this service account has no permission to mount the volume
153+
nodeSelector:
154+
"kubernetes.io/os": linux
155+
containers:
156+
- image: mcr.microsoft.com/oss/nginx/nginx:1.19.5
157+
name: nginx-blobfuse
158+
command:
159+
- "/bin/bash"
160+
- "-c"
161+
- set -euo pipefail; while true; do echo $(date) >> /mnt/blobfuse/outfile; sleep 1; done
162+
volumeMounts:
163+
- name: persistent-storage
164+
mountPath: "/mnt/blobfuse"
165+
readOnly: false
166+
volumes:
167+
- name: persistent-storage
168+
csi:
169+
driver: blob.csi.azure.com
170+
volumeAttributes:
171+
storageaccount: $ACCOUNT # required
172+
containerName: $CONTAINER # required
173+
clientID: $USER_ASSIGNED_CLIENT_ID # required
174+
resourcegroup: $STORAGE_RESOURCE_GROUP # optional, specified when the storage account is not under AKS node resource group(which is prefixed with "MC_")
175+
# tenantID: $IDENTITY_TENANT # optional, only specified when workload identity and AKS cluster are in different tenant
176+
# subscriptionid: $SUBSCRIPTION # optional, only specified when workload identity and AKS cluster are in different subscription
177+
EOF
178+
```

‎pkg/blob/blob.go

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,9 @@ const (
9696
requireInfraEncryptionField = "requireinfraencryption"
9797
ephemeralField = "csi.storage.k8s.io/ephemeral"
9898
podNamespaceField = "csi.storage.k8s.io/pod.namespace"
99+
serviceAccountTokenField = "csi.storage.k8s.io/serviceAccount.tokens"
100+
clientIDField = "clientID"
101+
tenantIDField = "tenantID"
99102
mountOptionsField = "mountoptions"
100103
falseValue = "false"
101104
trueValue = "true"
@@ -431,6 +434,9 @@ func (d *Driver) GetAuthEnv(ctx context.Context, volumeID, protocol string, attr
431434
authEnv []string
432435
getAccountKeyFromSecret bool
433436
getLatestAccountKey bool
437+
clientID string
438+
tenantID string
439+
serviceAccountToken string
434440
)
435441

436442
for k, v := range attrib {
@@ -480,6 +486,12 @@ func (d *Driver) GetAuthEnv(ctx context.Context, volumeID, protocol string, attr
480486
if getLatestAccountKey, err = strconv.ParseBool(v); err != nil {
481487
return rgName, accountName, accountKey, containerName, authEnv, fmt.Errorf("invalid %s: %s in volume context", getLatestAccountKeyField, v)
482488
}
489+
case strings.ToLower(clientIDField):
490+
clientID = v
491+
case strings.ToLower(tenantIDField):
492+
tenantID = v
493+
case strings.ToLower(serviceAccountTokenField):
494+
serviceAccountToken = v
483495
}
484496
}
485497
klog.V(2).Infof("volumeID(%s) authEnv: %s", volumeID, authEnv)
@@ -501,6 +513,21 @@ func (d *Driver) GetAuthEnv(ctx context.Context, volumeID, protocol string, attr
501513
rgName = d.cloud.ResourceGroup
502514
}
503515

516+
if tenantID == "" {
517+
tenantID = d.cloud.TenantID
518+
}
519+
520+
// if client id is specified, we only use service account token to get account key
521+
if clientID != "" {
522+
klog.V(2).Infof("clientID(%s) is specified, use service account token to get account key", clientID)
523+
if subsID == "" {
524+
subsID = d.cloud.SubscriptionID
525+
}
526+
accountKey, err := d.cloud.GetStorageAccesskeyFromServiceAccountToken(ctx, subsID, accountName, rgName, clientID, tenantID, serviceAccountToken)
527+
authEnv = append(authEnv, "AZURE_STORAGE_ACCESS_KEY="+accountKey)
528+
return rgName, accountName, accountKey, containerName, authEnv, err
529+
}
530+
504531
// 1. If keyVaultURL is not nil, preferentially use the key stored in key vault.
505532
// 2. Then if secrets map is not nil, use the key stored in the secrets map.
506533
// 3. Finally if both keyVaultURL and secrets map are nil, get the key from Azure.

‎pkg/blob/nodeserver.go

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,19 @@ func (d *Driver) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolu
8080
mountPermissions := d.mountPermissions
8181
context := req.GetVolumeContext()
8282
if context != nil {
83+
// token request
84+
if context[serviceAccountTokenField] != "" && getValueInMap(context, clientIDField) != "" {
85+
klog.V(2).Infof("NodePublishVolume: volume(%s) mount on %s with service account token, clientID: %s", volumeID, target, getValueInMap(context, clientIDField))
86+
_, err := d.NodeStageVolume(ctx, &csi.NodeStageVolumeRequest{
87+
StagingTargetPath: target,
88+
VolumeContext: context,
89+
VolumeCapability: volCap,
90+
VolumeId: volumeID,
91+
})
92+
return &csi.NodePublishVolumeResponse{}, err
93+
}
94+
95+
// ephemeral volume
8396
if strings.EqualFold(context[ephemeralField], trueValue) {
8497
setKeyValueInMap(context, secretNamespaceField, context[podNamespaceField])
8598
if !d.allowInlineVolumeKeyAccessWithIdentity {
@@ -239,6 +252,11 @@ func (d *Driver) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRe
239252
attrib := req.GetVolumeContext()
240253
secrets := req.GetSecrets()
241254

255+
if getValueInMap(attrib, clientIDField) != "" && attrib[serviceAccountTokenField] == "" {
256+
klog.V(2).Infof("Skip NodeStageVolume for volume(%s) since clientID %s is provided but service account token is empty", volumeID, getValueInMap(attrib, clientIDField))
257+
return &csi.NodeStageVolumeResponse{}, nil
258+
}
259+
242260
var serverAddress, storageEndpointSuffix, protocol, ephemeralVolMountOptions string
243261
var ephemeralVol, isHnsEnabled bool
244262

0 commit comments

Comments
 (0)