[RFC-0010] Add multi-tenant workload identity support for Azure Blob Storage

Signed-off-by: Dipti Pai <[email protected]>

Author: dipti-pai

api/v1/bucket_types.go

@@ -52,7 +52,7 @@ const (
 // +kubebuilder:validation:XValidation:rule="self.provider != 'generic' || !has(self.sts) || self.sts.provider == 'ldap'", message="'ldap' is the only supported STS provider for the 'generic' Bucket provider"
 // +kubebuilder:validation:XValidation:rule="!has(self.sts) || self.sts.provider != 'aws' || !has(self.sts.secretRef)", message="spec.sts.secretRef is not required for the 'aws' STS provider"
 // +kubebuilder:validation:XValidation:rule="!has(self.sts) || self.sts.provider != 'aws' || !has(self.sts.certSecretRef)", message="spec.sts.certSecretRef is not required for the 'aws' STS provider"
-// +kubebuilder:validation:XValidation:rule="self.provider == 'gcp' || self.provider == 'aws' || !has(self.serviceAccountName)", message="ServiceAccountName is only supported for the 'gcp' and 'aws' Bucket providers"
+// +kubebuilder:validation:XValidation:rule="self.provider != 'generic' || !has(self.serviceAccountName)", message="ServiceAccountName is not supported for the 'generic' Bucket provider"
 // +kubebuilder:validation:XValidation:rule="!has(self.secretRef) || !has(self.serviceAccountName)", message="cannot set both .spec.secretRef and .spec.serviceAccountName"
 type BucketSpec struct {
 	// Provider of the object storage bucket.

config/crd/bases/source.toolkit.fluxcd.io_buckets.yaml

@@ -239,9 +239,9 @@ spec:
               rule: '!has(self.sts) || self.sts.provider != ''aws'' || !has(self.sts.secretRef)'
             - message: spec.sts.certSecretRef is not required for the 'aws' STS provider
               rule: '!has(self.sts) || self.sts.provider != ''aws'' || !has(self.sts.certSecretRef)'
-            - message: ServiceAccountName is only supported for the 'gcp' and 'aws'
-                Bucket providers
-              rule: self.provider == 'gcp' || self.provider == 'aws' || !has(self.serviceAccountName)
+            - message: ServiceAccountName is not supported for the 'generic' Bucket
+                provider
+              rule: self.provider != 'generic' || !has(self.serviceAccountName)
             - message: cannot set both .spec.secretRef and .spec.serviceAccountName
               rule: '!has(self.secretRef) || !has(self.serviceAccountName)'
           status:

docs/spec/v1/buckets.md

@@ -567,83 +567,39 @@ metadata:
 spec:
   interval: 5m0s
   provider: azure
-  bucketName: testsas
-  endpoint: https://testfluxsas.blob.core.windows.net
+  bucketName: testwi
+  endpoint: https://testfluxwi.blob.core.windows.net
 ```
 
-##### Deprecated: Managed Identity with AAD Pod Identity
+##### Azure Object-Level Workload Identity example
 
-If you are using [aad pod identity](https://azure.github.io/aad-pod-identity/docs),
-You need to create an Azure Identity and give it access to Azure Blob Storage.
-
-```sh
-export IDENTITY_NAME="blob-access"
-
-az role assignment create --role "Storage Blob Data Reader"  \
---assignee-object-id "$(az identity show -n $IDENTITY_NAME -o tsv --query principalId  -g $RESOURCE_GROUP)" \
---scope "/subscriptions/<SUBSCRIPTION-ID>/resourceGroups/$RESOURCE_GROUP/providers/Microsoft.Storage/storageAccounts/<account-name>/blobServices/default/containers/<container-name>"
-
-export IDENTITY_CLIENT_ID="$(az identity show -n ${IDENTITY_NAME} -g ${RESOURCE_GROUP} -otsv --query clientId)"
-export IDENTITY_RESOURCE_ID="$(az identity show -n ${IDENTITY_NAME} -otsv --query id)"
-```
-
-Create an AzureIdentity object that references the identity created above:
+**Note:** To use Object-Level Workload Identity (`.spec.serviceAccountName` with 
+cloud providers), the controller feature gate `ObjectLevelWorkloadIdentity` must 
+be enabled.
 
 ```yaml
 ---
-apiVersion: aadpodidentity.k8s.io/v1
-kind: AzureIdentity
-metadata:
-  name:  # source-controller label will match this name
-  namespace: flux-system
-spec:
-  clientID: <IDENTITY_CLIENT_ID>
-  resourceID: <IDENTITY_RESOURCE_ID>
-  type: 0  # user-managed identity
-```
-
-Create an AzureIdentityBinding object that binds Pods with a specific selector
-with the AzureIdentity created:
-
-```yaml
-apiVersion: "aadpodidentity.k8s.io/v1"
-kind: AzureIdentityBinding
-metadata:
-  name: ${IDENTITY_NAME}-binding
-spec:
-  azureIdentity: ${IDENTITY_NAME}
-  selector: ${IDENTITY_NAME}
-```
-
-Label the source-controller Deployment correctly so that it can match an identity binding:
-
-```yaml
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: kustomize-controller
-  namespace: flux-system
-spec:
-  template:
-    metadata:
-      labels:
-        aadpodidbinding: ${IDENTITY_NAME}  # match the AzureIdentity name
-```
-
-If you have set up aad-pod-identity correctly and labeled the source-controller
-Deployment, then you don't need to reference a Secret.
-
-```yaml
 apiVersion: source.toolkit.fluxcd.io/v1
 kind: Bucket
 metadata:
-  name: azure-bucket
-  namespace: flux-system
+  name: azure-object-level-workload-identity
+  namespace: default
 spec:
   interval: 5m0s
   provider: azure
-  bucketName: testsas
-  endpoint: https://testfluxsas.blob.core.windows.net
+  bucketName: testwi
+  endpoint: https://testfluxwi.blob.core.windows.net
+  serviceAccountName: azure-workload-identity-sa
+  timeout: 30s
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: azure-workload-identity-sa
+  namespace: default
+  annotations:
+    azure.workload.identity/client-id: <AZURE_CLIENT_ID>
+    azure.workload.identity/tenant-id: <AZURE_TENANT_ID>
 ```
 
 ##### Azure Blob SAS Token example

internal/bucket/azure/blob.go

@@ -37,6 +37,8 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	ctrl "sigs.k8s.io/controller-runtime"
 
+	"github.com/fluxcd/pkg/auth"
+	azureauth "github.com/fluxcd/pkg/auth/azure"
 	"github.com/fluxcd/pkg/masktoken"
 
 	sourcev1 "github.com/fluxcd/source-controller/api/v1"
@@ -87,6 +89,7 @@ type options struct {
 	proxyURL           *url.URL
 	withoutCredentials bool
 	withoutRetries     bool
+	authOpts           []auth.Option
 }
 
 // withoutCredentials forces the BlobClient to not use any credentials.
@@ -107,6 +110,13 @@ func withoutRetries() Option {
 	}
 }
 
+// WithAuth sets the auth options for workload identity authentication.
+func WithAuth(authOpts ...auth.Option) Option {
+	return func(o *options) {
+		o.authOpts = authOpts
+	}
+}
+
 // NewClient creates a new Azure Blob storage client.
 // The credential config on the client is set based on the data from the
 // Bucket and Secret. It detects credentials in the Secret in the following
@@ -130,7 +140,7 @@ func withoutRetries() Option {
 //
 // If no credentials are found, and the azidentity.ChainedTokenCredential can
 // not be established. A simple client without credentials is returned.
-func NewClient(obj *sourcev1.Bucket, opts ...Option) (c *BlobClient, err error) {
+func NewClient(ctx context.Context, obj *sourcev1.Bucket, opts ...Option) (c *BlobClient, err error) {
 	c = &BlobClient{}
 
 	var o options
@@ -192,7 +202,7 @@ func NewClient(obj *sourcev1.Bucket, opts ...Option) (c *BlobClient, err error)
 	// Compose token chain based on environment.
 	// This functions as a replacement for azidentity.NewDefaultAzureCredential
 	// to not shell out.
-	token, err = chainCredentialWithSecret(o.secret)
+	token, err = chainCredentialWithSecret(ctx, o.secret, o.authOpts...)
 	if err != nil {
 		err = fmt.Errorf("failed to create environment credential chain: %w", err)
 		return nil, err
@@ -470,7 +480,7 @@ func sasTokenFromSecret(ep string, secret *corev1.Secret) (string, error) {
 //   - azidentity.ManagedIdentityCredential with defaults.
 //
 // If no valid token is created, it returns nil.
-func chainCredentialWithSecret(secret *corev1.Secret) (azcore.TokenCredential, error) {
+func chainCredentialWithSecret(ctx context.Context, secret *corev1.Secret, opts ...auth.Option) (azcore.TokenCredential, error) {
 	var creds []azcore.TokenCredential
 
 	credOpts := &azidentity.EnvironmentCredentialOptions{}
@@ -483,28 +493,7 @@ func chainCredentialWithSecret(secret *corev1.Secret) (azcore.TokenCredential, e
 	if token, _ := azidentity.NewEnvironmentCredential(credOpts); token != nil {
 		creds = append(creds, token)
 	}
-	if clientID := os.Getenv("AZURE_CLIENT_ID"); clientID != "" {
-		if file, ok := os.LookupEnv("AZURE_FEDERATED_TOKEN_FILE"); ok {
-			if _, ok := os.LookupEnv("AZURE_AUTHORITY_HOST"); ok {
-				if tenantID, ok := os.LookupEnv("AZURE_TENANT_ID"); ok {
-					if token, _ := azidentity.NewWorkloadIdentityCredential(&azidentity.WorkloadIdentityCredentialOptions{
-						ClientID:      clientID,
-						TenantID:      tenantID,
-						TokenFilePath: file,
-					}); token != nil {
-						creds = append(creds, token)
-					}
-				}
-			}
-		}
-
-		if token, _ := azidentity.NewManagedIdentityCredential(&azidentity.ManagedIdentityCredentialOptions{
-			ID: azidentity.ClientID(clientID),
-		}); token != nil {
-			creds = append(creds, token)
-		}
-	}
-	if token, _ := azidentity.NewManagedIdentityCredential(nil); token != nil {
+	if token := azureauth.NewTokenCredential(ctx, opts...); token != nil {
 		creds = append(creds, token)
 	}
 

internal/bucket/azure/blob_test.go

@@ -106,7 +106,8 @@ func TestNewClientAndBucketExistsWithProxy(t *testing.T) {
 				},
 			}
 
-			client, err := NewClient(bucket,
+			client, err := NewClient(t.Context(),
+				bucket,
 				WithProxyURL(tt.proxyURL),
 				withoutCredentials(),
 				withoutRetries())
@@ -472,7 +473,7 @@ func Test_sasTokenFromSecret(t *testing.T) {
 func Test_chainCredentialWithSecret(t *testing.T) {
 	g := NewWithT(t)
 
-	got, err := chainCredentialWithSecret(nil)
+	got, err := chainCredentialWithSecret(t.Context(), nil, nil)
 	g.Expect(err).ToNot(HaveOccurred())
 	g.Expect(got).To(BeAssignableToTypeOf(&azidentity.ChainedTokenCredential{}))
 }

internal/controller/bucket_controller.go

@@ -920,7 +920,8 @@ func (r *BucketReconciler) createBucketProvider(ctx context.Context, obj *source
 		if creds.proxyURL != nil {
 			opts = append(opts, azure.WithProxyURL(creds.proxyURL))
 		}
-		return azure.NewClient(obj, opts...)
+		opts = append(opts, azure.WithAuth(authOpts...))
+		return azure.NewClient(ctx, obj, opts...)
 
 	default:
 		if err := minio.ValidateSecret(creds.secret); err != nil {