|
| 1 | +# Example of static PV mount with workload identity |
| 2 | + |
| 3 | +> Note: |
| 4 | +> - Available kubernetes version >= v1.20 |
| 5 | +
|
| 6 | +## prerequisite |
| 7 | + |
| 8 | + |
| 9 | +### 1. Create a cluster with oidc-issuer enabled and get the credential |
| 10 | + |
| 11 | +Following the [documentation](https://learn.microsoft.com/en-us/azure/aks/use-oidc-issuer#create-an-aks-cluster-with-oidc-issuer) to create an AKS cluster with the `--enable-oidc-issuer` parameter and get the AKS credentials. And export following environment variables: |
| 12 | +``` |
| 13 | +export RESOURCE_GROUP=<your resource group name> |
| 14 | +export CLUSTER_NAME=<your cluster name> |
| 15 | +export REGION=<your region> |
| 16 | +``` |
| 17 | + |
| 18 | + |
| 19 | +### 2. Create a new storage account and fileshare |
| 20 | + |
| 21 | +Following the [documentation](https://learn.microsoft.com/en-us/azure/storage/blobs/storage-quickstart-blobs-cli) to create a new storage account and container or use your own. And export following environment variables: |
| 22 | +``` |
| 23 | +export STORAGE_RESOURCE_GROUP=<your storage account resource group> |
| 24 | +export ACCOUNT=<your storage account name> |
| 25 | +export CONTAINER=<your container name> |
| 26 | +``` |
| 27 | + |
| 28 | +### 3. Create managed identity and role assignment |
| 29 | +``` |
| 30 | +export UAMI=<your managed identity name> |
| 31 | +az identity create --name $UAMI --resource-group $RESOURCE_GROUP |
| 32 | +
|
| 33 | +export USER_ASSIGNED_CLIENT_ID="$(az identity show -g $RESOURCE_GROUP --name $UAMI --query 'clientId' -o tsv)" |
| 34 | +export IDENTITY_TENANT=$(az aks show --name $CLUSTER_NAME --resource-group $RESOURCE_GROUP --query identity.tenantId -o tsv) |
| 35 | +export ACCOUNT_SCOPE=$(az storage account show --name $ACCOUNT --query id -o tsv) |
| 36 | +
|
| 37 | +# please retry if you meet `Cannot find user or service principal in graph database` error, it may take a while for the identity to propagate |
| 38 | +az role assignment create --role "Storage Account Contributor" --assignee $USER_ASSIGNED_CLIENT_ID --scope $ACCOUNT_SCOPE |
| 39 | +``` |
| 40 | + |
| 41 | +### 4. Create service account on AKS |
| 42 | +``` |
| 43 | +export SERVICE_ACCOUNT_NAME=<your sa name> |
| 44 | +export SERVICE_ACCOUNT_NAMESPACE=<your sa namespace> |
| 45 | +
|
| 46 | +cat <<EOF | kubectl apply -f - |
| 47 | +apiVersion: v1 |
| 48 | +kind: ServiceAccount |
| 49 | +metadata: |
| 50 | + name: ${SERVICE_ACCOUNT_NAME} |
| 51 | + namespace: ${SERVICE_ACCOUNT_NAMESPACE} |
| 52 | +EOF |
| 53 | +``` |
| 54 | + |
| 55 | +### 5. Create the federated identity credential between the managed identity, service account issuer, and subject using the `az identity federated-credential create` command. |
| 56 | +``` |
| 57 | +export FEDERATED_IDENTITY_NAME=<your federated identity name> |
| 58 | +export AKS_OIDC_ISSUER="$(az aks show --resource-group $RESOURCE_GROUP --name $CLUSTER_NAME --query "oidcIssuerProfile.issuerUrl" -o tsv)" |
| 59 | +
|
| 60 | +az identity federated-credential create --name $FEDERATED_IDENTITY_NAME \ |
| 61 | +--identity-name $UAMI \ |
| 62 | +--resource-group $RESOURCE_GROUP \ |
| 63 | +--issuer $AKS_OIDC_ISSUER \ |
| 64 | +--subject system:serviceaccount:${SERVICE_ACCOUNT_NAMESPACE}:${SERVICE_ACCOUNT_NAME} |
| 65 | +``` |
| 66 | + |
| 67 | +## option#1: static provision with PV |
| 68 | +``` |
| 69 | +cat <<EOF | kubectl apply -f - |
| 70 | +apiVersion: v1 |
| 71 | +kind: PersistentVolume |
| 72 | +metadata: |
| 73 | + annotations: |
| 74 | + pv.kubernetes.io/provisioned-by: blob.csi.azure.com |
| 75 | + name: pv-blob |
| 76 | +spec: |
| 77 | + capacity: |
| 78 | + storage: 10Gi |
| 79 | + accessModes: |
| 80 | + - ReadWriteMany |
| 81 | + persistentVolumeReclaimPolicy: Retain |
| 82 | + storageClassName: blob-fuse |
| 83 | + mountOptions: |
| 84 | + - -o allow_other |
| 85 | + - --file-cache-timeout-in-seconds=120 |
| 86 | + csi: |
| 87 | + driver: blob.csi.azure.com |
| 88 | + # make sure volumeid is unique for every storage blob container in the cluster |
| 89 | + # the # character is reserved for internal use, the / character is not allowed |
| 90 | + volumeHandle: unique_volume_id |
| 91 | + volumeAttributes: |
| 92 | + storageaccount: $ACCOUNT # required |
| 93 | + containerName: $CONTAINER # required |
| 94 | + clientID: $USER_ASSIGNED_CLIENT_ID # required |
| 95 | + resourcegroup: $STORAGE_RESOURCE_GROUP # optional, specified when the storage account is not under AKS node resource group(which is prefixed with "MC_") |
| 96 | + # tenantID: $IDENTITY_TENANT #optional, only specified when workload identity and AKS cluster are in different tenant |
| 97 | + # subscriptionid: $SUBSCRIPTION #optional, only specified when workload identity and AKS cluster are in different subscription |
| 98 | +--- |
| 99 | +apiVersion: apps/v1 |
| 100 | +kind: StatefulSet |
| 101 | +metadata: |
| 102 | + name: statefulset-blob |
| 103 | + labels: |
| 104 | + app: nginx |
| 105 | +spec: |
| 106 | + serviceName: statefulset-blob |
| 107 | + replicas: 1 |
| 108 | + template: |
| 109 | + metadata: |
| 110 | + labels: |
| 111 | + app: nginx |
| 112 | + spec: |
| 113 | + serviceAccountName: $SERVICE_ACCOUNT_NAME #required, Pod does not use this service account has no permission to mount the volume |
| 114 | + nodeSelector: |
| 115 | + "kubernetes.io/os": linux |
| 116 | + containers: |
| 117 | + - name: statefulset-blob |
| 118 | + image: mcr.microsoft.com/oss/nginx/nginx:1.19.5 |
| 119 | + command: |
| 120 | + - "/bin/bash" |
| 121 | + - "-c" |
| 122 | + - set -euo pipefail; while true; do echo $(date) >> /mnt/blob/outfile; sleep 1; done |
| 123 | + volumeMounts: |
| 124 | + - name: persistent-storage |
| 125 | + mountPath: /mnt/blob |
| 126 | + readOnly: false |
| 127 | + updateStrategy: |
| 128 | + type: RollingUpdate |
| 129 | + selector: |
| 130 | + matchLabels: |
| 131 | + app: nginx |
| 132 | + volumeClaimTemplates: |
| 133 | + - metadata: |
| 134 | + name: persistent-storage |
| 135 | + spec: |
| 136 | + storageClassName: blob-fuse |
| 137 | + accessModes: ["ReadWriteMany"] |
| 138 | + resources: |
| 139 | + requests: |
| 140 | + storage: 10Gi |
| 141 | +EOF |
| 142 | +``` |
| 143 | + |
| 144 | +## option#2: Pod with ephemeral inline volume |
| 145 | +``` |
| 146 | +cat <<EOF | kubectl apply -f - |
| 147 | +kind: Pod |
| 148 | +apiVersion: v1 |
| 149 | +metadata: |
| 150 | + name: nginx-blobfuse-inline-volume |
| 151 | +spec: |
| 152 | + serviceAccountName: $SERVICE_ACCOUNT_NAME #required, Pod does not use this service account has no permission to mount the volume |
| 153 | + nodeSelector: |
| 154 | + "kubernetes.io/os": linux |
| 155 | + containers: |
| 156 | + - image: mcr.microsoft.com/oss/nginx/nginx:1.19.5 |
| 157 | + name: nginx-blobfuse |
| 158 | + command: |
| 159 | + - "/bin/bash" |
| 160 | + - "-c" |
| 161 | + - set -euo pipefail; while true; do echo $(date) >> /mnt/blobfuse/outfile; sleep 1; done |
| 162 | + volumeMounts: |
| 163 | + - name: persistent-storage |
| 164 | + mountPath: "/mnt/blobfuse" |
| 165 | + readOnly: false |
| 166 | + volumes: |
| 167 | + - name: persistent-storage |
| 168 | + csi: |
| 169 | + driver: blob.csi.azure.com |
| 170 | + volumeAttributes: |
| 171 | + storageaccount: $ACCOUNT # required |
| 172 | + containerName: $CONTAINER # required |
| 173 | + clientID: $USER_ASSIGNED_CLIENT_ID # required |
| 174 | + resourcegroup: $STORAGE_RESOURCE_GROUP # optional, specified when the storage account is not under AKS node resource group(which is prefixed with "MC_") |
| 175 | + # tenantID: $IDENTITY_TENANT # optional, only specified when workload identity and AKS cluster are in different tenant |
| 176 | + # subscriptionid: $SUBSCRIPTION # optional, only specified when workload identity and AKS cluster are in different subscription |
| 177 | +EOF |
| 178 | +``` |
0 commit comments