Skip to content

Commit 4183155

Browse files
authored
Merge pull request #1211 from cvvz/release-1.23-1204
[release-1.23] feat: support workload identity setting in static PV mount
2 parents 60e4168 + 4b1f2d1 commit 4183155

File tree

1,025 files changed

+299384
-21053
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

1,025 files changed

+299384
-21053
lines changed

charts/latest/blob-csi-driver/templates/csi-blob-driver.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,3 +12,5 @@ spec:
1212
volumeLifecycleModes:
1313
- Persistent
1414
- Ephemeral
15+
tokenRequests:
16+
- audience: api://AzureADTokenExchange
File renamed without changes.
Lines changed: 178 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,178 @@
1+
# Example of static PV mount with workload identity
2+
3+
> Note:
4+
> - Available kubernetes version >= v1.20
5+
6+
## prerequisite
7+
8+
9+
### 1. Create a cluster with oidc-issuer enabled and get the credential
10+
11+
Following the [documentation](https://learn.microsoft.com/en-us/azure/aks/use-oidc-issuer#create-an-aks-cluster-with-oidc-issuer) to create an AKS cluster with the `--enable-oidc-issuer` parameter and get the AKS credentials. And export following environment variables:
12+
```
13+
export RESOURCE_GROUP=<your resource group name>
14+
export CLUSTER_NAME=<your cluster name>
15+
export REGION=<your region>
16+
```
17+
18+
19+
### 2. Create a new storage account and container
20+
21+
Following the [documentation](https://learn.microsoft.com/en-us/azure/storage/blobs/storage-quickstart-blobs-cli) to create a new storage account and container or use your own. And export following environment variables:
22+
```
23+
export STORAGE_RESOURCE_GROUP=<your storage account resource group>
24+
export ACCOUNT=<your storage account name>
25+
export CONTAINER=<your container name>
26+
```
27+
28+
### 3. Create managed identity and role assignment
29+
```
30+
export UAMI=<your managed identity name>
31+
az identity create --name $UAMI --resource-group $RESOURCE_GROUP
32+
33+
export USER_ASSIGNED_CLIENT_ID="$(az identity show -g $RESOURCE_GROUP --name $UAMI --query 'clientId' -o tsv)"
34+
export IDENTITY_TENANT=$(az aks show --name $CLUSTER_NAME --resource-group $RESOURCE_GROUP --query identity.tenantId -o tsv)
35+
export ACCOUNT_SCOPE=$(az storage account show --name $ACCOUNT --query id -o tsv)
36+
37+
# please retry if you meet `Cannot find user or service principal in graph database` error, it may take a while for the identity to propagate
38+
az role assignment create --role "Storage Account Contributor" --assignee $USER_ASSIGNED_CLIENT_ID --scope $ACCOUNT_SCOPE
39+
```
40+
41+
### 4. Create service account on AKS
42+
```
43+
export SERVICE_ACCOUNT_NAME=<your sa name>
44+
export SERVICE_ACCOUNT_NAMESPACE=<your sa namespace>
45+
46+
cat <<EOF | kubectl apply -f -
47+
apiVersion: v1
48+
kind: ServiceAccount
49+
metadata:
50+
name: ${SERVICE_ACCOUNT_NAME}
51+
namespace: ${SERVICE_ACCOUNT_NAMESPACE}
52+
EOF
53+
```
54+
55+
### 5. Create the federated identity credential between the managed identity, service account issuer, and subject using the `az identity federated-credential create` command.
56+
```
57+
export FEDERATED_IDENTITY_NAME=<your federated identity name>
58+
export AKS_OIDC_ISSUER="$(az aks show --resource-group $RESOURCE_GROUP --name $CLUSTER_NAME --query "oidcIssuerProfile.issuerUrl" -o tsv)"
59+
60+
az identity federated-credential create --name $FEDERATED_IDENTITY_NAME \
61+
--identity-name $UAMI \
62+
--resource-group $RESOURCE_GROUP \
63+
--issuer $AKS_OIDC_ISSUER \
64+
--subject system:serviceaccount:${SERVICE_ACCOUNT_NAMESPACE}:${SERVICE_ACCOUNT_NAME}
65+
```
66+
67+
## option#1: static provision with PV
68+
```
69+
cat <<EOF | kubectl apply -f -
70+
apiVersion: v1
71+
kind: PersistentVolume
72+
metadata:
73+
annotations:
74+
pv.kubernetes.io/provisioned-by: blob.csi.azure.com
75+
name: pv-blob
76+
spec:
77+
capacity:
78+
storage: 10Gi
79+
accessModes:
80+
- ReadWriteMany
81+
persistentVolumeReclaimPolicy: Retain
82+
storageClassName: blob-fuse
83+
mountOptions:
84+
- -o allow_other
85+
- --file-cache-timeout-in-seconds=120
86+
csi:
87+
driver: blob.csi.azure.com
88+
# make sure volumeid is unique for every storage blob container in the cluster
89+
# the # character is reserved for internal use, the / character is not allowed
90+
volumeHandle: unique_volume_id
91+
volumeAttributes:
92+
storageaccount: $ACCOUNT # required
93+
containerName: $CONTAINER # required
94+
clientID: $USER_ASSIGNED_CLIENT_ID # required
95+
resourcegroup: $STORAGE_RESOURCE_GROUP # optional, specified when the storage account is not under AKS node resource group(which is prefixed with "MC_")
96+
# tenantID: $IDENTITY_TENANT #optional, only specified when workload identity and AKS cluster are in different tenant
97+
# subscriptionid: $SUBSCRIPTION #optional, only specified when workload identity and AKS cluster are in different subscription
98+
---
99+
apiVersion: apps/v1
100+
kind: StatefulSet
101+
metadata:
102+
name: statefulset-blob
103+
labels:
104+
app: nginx
105+
spec:
106+
serviceName: statefulset-blob
107+
replicas: 1
108+
template:
109+
metadata:
110+
labels:
111+
app: nginx
112+
spec:
113+
serviceAccountName: $SERVICE_ACCOUNT_NAME #required, Pod does not use this service account has no permission to mount the volume
114+
nodeSelector:
115+
"kubernetes.io/os": linux
116+
containers:
117+
- name: statefulset-blob
118+
image: mcr.microsoft.com/oss/nginx/nginx:1.19.5
119+
command:
120+
- "/bin/bash"
121+
- "-c"
122+
- set -euo pipefail; while true; do echo $(date) >> /mnt/blob/outfile; sleep 1; done
123+
volumeMounts:
124+
- name: persistent-storage
125+
mountPath: /mnt/blob
126+
readOnly: false
127+
updateStrategy:
128+
type: RollingUpdate
129+
selector:
130+
matchLabels:
131+
app: nginx
132+
volumeClaimTemplates:
133+
- metadata:
134+
name: persistent-storage
135+
spec:
136+
storageClassName: blob-fuse
137+
accessModes: ["ReadWriteMany"]
138+
resources:
139+
requests:
140+
storage: 10Gi
141+
EOF
142+
```
143+
144+
## option#2: Pod with ephemeral inline volume
145+
```
146+
cat <<EOF | kubectl apply -f -
147+
kind: Pod
148+
apiVersion: v1
149+
metadata:
150+
name: nginx-blobfuse-inline-volume
151+
spec:
152+
serviceAccountName: $SERVICE_ACCOUNT_NAME #required, Pod does not use this service account has no permission to mount the volume
153+
nodeSelector:
154+
"kubernetes.io/os": linux
155+
containers:
156+
- image: mcr.microsoft.com/oss/nginx/nginx:1.19.5
157+
name: nginx-blobfuse
158+
command:
159+
- "/bin/bash"
160+
- "-c"
161+
- set -euo pipefail; while true; do echo $(date) >> /mnt/blobfuse/outfile; sleep 1; done
162+
volumeMounts:
163+
- name: persistent-storage
164+
mountPath: "/mnt/blobfuse"
165+
readOnly: false
166+
volumes:
167+
- name: persistent-storage
168+
csi:
169+
driver: blob.csi.azure.com
170+
volumeAttributes:
171+
storageaccount: $ACCOUNT # required
172+
containerName: $CONTAINER # required
173+
clientID: $USER_ASSIGNED_CLIENT_ID # required
174+
resourcegroup: $STORAGE_RESOURCE_GROUP # optional, specified when the storage account is not under AKS node resource group(which is prefixed with "MC_")
175+
# tenantID: $IDENTITY_TENANT # optional, only specified when workload identity and AKS cluster are in different tenant
176+
# subscriptionid: $SUBSCRIPTION # optional, only specified when workload identity and AKS cluster are in different subscription
177+
EOF
178+
```

go.mod

Lines changed: 49 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,12 @@
11
module sigs.k8s.io/blob-csi-driver
22

3-
go 1.20
3+
go 1.21
4+
5+
toolchain go1.21.4
46

57
require (
68
github.com/Azure/azure-sdk-for-go v68.0.0+incompatible
7-
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0
9+
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0
810
github.com/Azure/go-autorest/autorest v0.11.29
911
github.com/Azure/go-autorest/autorest/adal v0.9.23
1012
github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect
@@ -14,7 +16,7 @@ require (
1416
github.com/golang/protobuf v1.5.3
1517
github.com/imdario/mergo v0.3.9 // indirect
1618
github.com/kubernetes-csi/csi-lib-utils v0.13.0
17-
github.com/onsi/gomega v1.27.10
19+
github.com/onsi/gomega v1.30.0
1820
github.com/pborman/uuid v1.2.1
1921
github.com/pelletier/go-toml v1.9.5
2022
github.com/stretchr/testify v1.8.4
@@ -25,36 +27,43 @@ require (
2527
k8s.io/apimachinery v0.28.4
2628
k8s.io/client-go v0.28.4
2729
k8s.io/component-base v0.28.4
28-
k8s.io/klog/v2 v2.100.1
30+
k8s.io/klog/v2 v2.110.1
2931
k8s.io/kubernetes v1.28.4
3032
k8s.io/mount-utils v0.28.4
31-
k8s.io/utils v0.0.0-20230406110748-d93618cff8a2
32-
sigs.k8s.io/cloud-provider-azure v1.27.1-0.20230907063607-e9994a5f9c7a
33-
sigs.k8s.io/yaml v1.3.0
33+
k8s.io/utils v0.0.0-20231127182322-b307cd553661
34+
sigs.k8s.io/cloud-provider-azure v1.27.1-0.20231213062409-f1ce7de3fdcb
35+
sigs.k8s.io/yaml v1.4.0
3436
)
3537

3638
require (
37-
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0
38-
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.0.0
39-
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.6.1
39+
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.0
40+
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.4.0
41+
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0
4042
github.com/go-ini/ini v1.67.0
41-
github.com/jongio/azidext/go/azidext v0.4.0
42-
github.com/onsi/ginkgo/v2 v2.11.0
43+
github.com/onsi/ginkgo/v2 v2.13.2
4344
github.com/pkg/errors v0.9.1
4445
github.com/satori/go.uuid v1.2.0
4546
k8s.io/apiserver v0.28.4
4647
k8s.io/pod-security-admission v0.28.4
48+
sigs.k8s.io/cloud-provider-azure/pkg/azclient/configloader v0.0.0-20231205023417-1ba5a224ab0e
4749
)
4850

4951
require (
50-
github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect
52+
github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.0 // indirect
53+
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.3.0 // indirect
54+
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerregistry/armcontainerregistry v1.2.0 // indirect
55+
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v4 v4.6.0 // indirect
56+
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4 v4.3.0 // indirect
57+
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/privatedns/armprivatedns v1.2.0 // indirect
58+
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0
59+
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0 // indirect
5160
github.com/Azure/go-autorest v14.2.0+incompatible // indirect
5261
github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
5362
github.com/Azure/go-autorest/autorest/mocks v0.4.2 // indirect
5463
github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect
5564
github.com/Azure/go-autorest/logger v0.2.1 // indirect
5665
github.com/Azure/go-autorest/tracing v0.6.0 // indirect
57-
github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 // indirect
66+
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.0 // indirect
5867
github.com/NYTimes/gziphandler v1.1.1 // indirect
5968
github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df // indirect
6069
github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect
@@ -66,27 +75,28 @@ require (
6675
github.com/coreos/go-systemd/v22 v22.5.0 // indirect
6776
github.com/davecgh/go-spew v1.1.1 // indirect
6877
github.com/docker/distribution v2.8.2+incompatible // indirect
69-
github.com/emicklei/go-restful/v3 v3.9.0 // indirect
70-
github.com/evanphx/json-patch v5.6.0+incompatible // indirect
71-
github.com/felixge/httpsnoop v1.0.3 // indirect
72-
github.com/fsnotify/fsnotify v1.6.0 // indirect
73-
github.com/go-logr/logr v1.2.4 // indirect
78+
github.com/emicklei/go-restful/v3 v3.10.2 // indirect
79+
github.com/evanphx/json-patch v5.7.0+incompatible // indirect
80+
github.com/felixge/httpsnoop v1.0.4 // indirect
81+
github.com/fsnotify/fsnotify v1.7.0 // indirect
82+
github.com/go-logr/logr v1.3.0 // indirect
7483
github.com/go-logr/stdr v1.2.2 // indirect
7584
github.com/go-openapi/jsonpointer v0.19.6 // indirect
7685
github.com/go-openapi/jsonreference v0.20.2 // indirect
77-
github.com/go-openapi/swag v0.22.3 // indirect
86+
github.com/go-openapi/swag v0.22.4 // indirect
7887
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
7988
github.com/gogo/protobuf v1.3.2 // indirect
8089
github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
90+
github.com/golang-jwt/jwt/v5 v5.0.0 // indirect
8191
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
8292
github.com/google/cel-go v0.16.1 // indirect
8393
github.com/google/gnostic-models v0.6.8 // indirect
84-
github.com/google/go-cmp v0.5.9 // indirect
94+
github.com/google/go-cmp v0.6.0 // indirect
8595
github.com/google/gofuzz v1.2.0 // indirect
86-
github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect
87-
github.com/google/uuid v1.3.1 // indirect
96+
github.com/google/pprof v0.0.0-20230602010524-ada837c32108 // indirect
97+
github.com/google/uuid v1.4.0 // indirect
8898
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
89-
github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect
99+
github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect
90100
github.com/inconshreveable/mousetrap v1.1.0 // indirect
91101
github.com/josharian/intern v1.0.0 // indirect
92102
github.com/json-iterator/go v1.1.12 // indirect
@@ -106,34 +116,33 @@ require (
106116
github.com/prometheus/client_model v0.4.0 // indirect
107117
github.com/prometheus/common v0.44.0 // indirect
108118
github.com/prometheus/procfs v0.10.1 // indirect
109-
github.com/spf13/cobra v1.7.0 // indirect
119+
github.com/spf13/cobra v1.8.0 // indirect
110120
github.com/spf13/pflag v1.0.5 // indirect
111121
github.com/stoewer/go-strcase v1.2.0 // indirect
112122
go.etcd.io/etcd/api/v3 v3.5.9 // indirect
113123
go.etcd.io/etcd/client/pkg/v3 v3.5.9 // indirect
114124
go.etcd.io/etcd/client/v3 v3.5.9 // indirect
115-
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.35.0 // indirect
116-
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.35.1 // indirect
117-
go.opentelemetry.io/otel v1.10.0 // indirect
118-
go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.10.0 // indirect
119-
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.10.0 // indirect
120-
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.10.0 // indirect
121-
go.opentelemetry.io/otel/metric v0.31.0 // indirect
122-
go.opentelemetry.io/otel/sdk v1.10.0 // indirect
123-
go.opentelemetry.io/otel/trace v1.10.0 // indirect
124-
go.opentelemetry.io/proto/otlp v0.19.0 // indirect
125+
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.0 // indirect
126+
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.0 // indirect
127+
go.opentelemetry.io/otel v1.20.0 // indirect
128+
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.20.0 // indirect
129+
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.20.0 // indirect
130+
go.opentelemetry.io/otel/metric v1.20.0 // indirect
131+
go.opentelemetry.io/otel/sdk v1.20.0 // indirect
132+
go.opentelemetry.io/otel/trace v1.20.0 // indirect
133+
go.opentelemetry.io/proto/otlp v1.0.0 // indirect
125134
go.uber.org/atomic v1.10.0 // indirect
126135
go.uber.org/multierr v1.11.0 // indirect
127136
go.uber.org/zap v1.19.0 // indirect
128137
golang.org/x/crypto v0.17.0 // indirect
129-
golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect
138+
golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
130139
golang.org/x/oauth2 v0.11.0 // indirect
131-
golang.org/x/sync v0.3.0 // indirect
140+
golang.org/x/sync v0.5.0
132141
golang.org/x/sys v0.15.0 // indirect
133142
golang.org/x/term v0.15.0 // indirect
134143
golang.org/x/text v0.14.0 // indirect
135-
golang.org/x/time v0.3.0 // indirect
136-
golang.org/x/tools v0.9.3 // indirect
144+
golang.org/x/time v0.5.0 // indirect
145+
golang.org/x/tools v0.14.0 // indirect
137146
google.golang.org/appengine v1.6.7 // indirect
138147
google.golang.org/genproto v0.0.0-20231030173426-d783a09b4405 // indirect
139148
google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b // indirect
@@ -151,6 +160,7 @@ require (
151160
k8s.io/kubectl v0.0.0 // indirect
152161
k8s.io/kubelet v0.28.4 // indirect
153162
sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.2 // indirect
163+
sigs.k8s.io/cloud-provider-azure/pkg/azclient v0.0.0-20231205023417-1ba5a224ab0e
154164
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
155165
sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
156166
)

0 commit comments

Comments
 (0)