Merge pull request #5000 from nojnhuh/capi-self-hosted

use managed identity for CAPI self-hosted e2e test

Author: k8s-ci-robot

azure/scope/clients.go

@@ -27,6 +27,7 @@ import (
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
 	azureautorest "github.com/Azure/go-autorest/autorest/azure"
 	"github.com/Azure/go-autorest/autorest/azure/auth"
+	infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1"
 )
 
 // AzureClients contains all the Azure clients used by the scopes.
@@ -36,6 +37,8 @@ type AzureClients struct {
 	TokenCredential            azcore.TokenCredential
 	ResourceManagerEndpoint    string
 	ResourceManagerVMDNSSuffix string
+
+	authType infrav1.IdentityType
 }
 
 // CloudEnvironment returns the Azure environment the controller runs in.
@@ -73,7 +76,7 @@ func (c *AzureClients) Token() azcore.TokenCredential {
 // ClientID).
 func (c *AzureClients) HashKey() string {
 	hasher := sha256.New()
-	_, _ = hasher.Write([]byte(c.TenantID() + c.CloudEnvironment() + c.SubscriptionID() + c.ClientID()))
+	_, _ = hasher.Write([]byte(c.TenantID() + c.CloudEnvironment() + c.SubscriptionID() + c.ClientID() + string(c.authType)))
 	return base64.URLEncoding.EncodeToString(hasher.Sum(nil))
 }
 
@@ -107,6 +110,8 @@ func (c *AzureClients) setCredentialsWithProvider(ctx context.Context, subscript
 	}
 	c.Values["AZURE_CLIENT_SECRET"] = strings.TrimSuffix(clientSecret, "\n")
 
+	c.authType = credentialsProvider.Type()
+
 	tokenCredential, err := credentialsProvider.GetTokenCredential(ctx, c.ResourceManagerEndpoint, c.Environment.ActiveDirectoryEndpoint, c.Environment.TokenAudience)
 	if err != nil {
 		return err

azure/scope/identity.go

@@ -41,6 +41,7 @@ type CredentialsProvider interface {
 	GetClientSecret(ctx context.Context) (string, error)
 	GetTenantID() string
 	GetTokenCredential(ctx context.Context, resourceManagerEndpoint, activeDirectoryEndpoint, tokenAudience string) (azcore.TokenCredential, error)
+	Type() infrav1.IdentityType
 }
 
 // AzureCredentialsProvider represents a credential provider with azure cluster identity.
@@ -228,6 +229,11 @@ func (p *AzureCredentialsProvider) GetTenantID() string {
 	return p.Identity.Spec.TenantID
 }
 
+// Type returns the auth mechanism used.
+func (p *AzureCredentialsProvider) Type() infrav1.IdentityType {
+	return p.Identity.Spec.Type
+}
+
 // hasClientSecret returns true if the identity has a Service Principal Client Secret.
 // This does not include managed identities.
 func (p *AzureCredentialsProvider) hasClientSecret() bool {

test/e2e/azure_selfhosted.go

@@ -25,10 +25,15 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
+	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
+	clusterctlv1 "sigs.k8s.io/cluster-api/cmd/clusterctl/api/v1alpha3"
 	capi_e2e "sigs.k8s.io/cluster-api/test/e2e"
 	"sigs.k8s.io/cluster-api/test/framework"
 	"sigs.k8s.io/cluster-api/test/framework/clusterctl"
@@ -141,6 +146,40 @@ func SelfHostedSpec(ctx context.Context, inputGetter func() SelfHostedSpecInput)
 			Namespace:            namespace.Name,
 		})
 
+		// The workload cluster is not set up for workload identity. Use UserAssignedMSI there instead.
+		err := selfHostedClusterProxy.GetClient().Delete(ctx, &infrav1.AzureClusterIdentity{
+			ObjectMeta: metav1.ObjectMeta{
+				Namespace: cluster.Namespace,
+				Name:      e2eConfig.GetVariable(ClusterIdentityName),
+			},
+		})
+		Expect(err).NotTo(HaveOccurred())
+		cred, err := azidentity.NewDefaultAzureCredential(nil)
+		Expect(err).NotTo(HaveOccurred())
+		identityClient, err := armmsi.NewUserAssignedIdentitiesClient(getSubscriptionID(Default), cred, nil)
+		Expect(err).NotTo(HaveOccurred())
+		identityRG := e2eConfig.GetVariable(AzureIdentityResourceGroup)
+		identityName := e2eConfig.GetVariable(AzureUserIdentity)
+		identity, err := identityClient.Get(ctx, identityRG, identityName, nil)
+		Expect(err).NotTo(HaveOccurred())
+		err = selfHostedClusterProxy.GetClient().Create(ctx, &infrav1.AzureClusterIdentity{
+			ObjectMeta: metav1.ObjectMeta{
+				Namespace: cluster.Namespace,
+				Name:      e2eConfig.GetVariable(ClusterIdentityName),
+				Labels: map[string]string{
+					clusterctlv1.ClusterctlMoveHierarchyLabel: "true",
+				},
+			},
+			Spec: infrav1.AzureClusterIdentitySpec{
+				AllowedNamespaces: &infrav1.AllowedNamespaces{},
+				ClientID:          *identity.Properties.ClientID,
+				ResourceID:        *identity.ID,
+				TenantID:          e2eConfig.GetVariable(AzureTenantID),
+				Type:              infrav1.UserAssignedMSI,
+			},
+		})
+		Expect(err).NotTo(HaveOccurred())
+
 		Log("Waiting for the cluster to be reconciled after moving to self hosted")
 		selfHostedCluster = framework.DiscoveryAndWaitForCluster(ctx, framework.DiscoveryAndWaitForClusterInput{
 			Getter:    selfHostedClusterProxy.GetClient(),
@@ -193,6 +232,31 @@ func SelfHostedSpec(ctx context.Context, inputGetter func() SelfHostedSpecInput)
 				Namespace:            selfHostedNamespace.Name,
 			})
 
+			// Restore the workload identity AzureClusterIdentity
+			err := input.BootstrapClusterProxy.GetClient().Delete(ctx, &infrav1.AzureClusterIdentity{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: namespace.Name,
+					Name:      e2eConfig.GetVariable(ClusterIdentityName),
+				},
+			})
+			Expect(err).NotTo(HaveOccurred())
+			err = input.BootstrapClusterProxy.GetClient().Create(ctx, &infrav1.AzureClusterIdentity{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: namespace.Name,
+					Name:      e2eConfig.GetVariable(ClusterIdentityName),
+					Labels: map[string]string{
+						clusterctlv1.ClusterctlMoveHierarchyLabel: "true",
+					},
+				},
+				Spec: infrav1.AzureClusterIdentitySpec{
+					AllowedNamespaces: &infrav1.AllowedNamespaces{},
+					ClientID:          e2eConfig.GetVariable(AzureClientIDUserAssignedIdentity),
+					TenantID:          e2eConfig.GetVariable(AzureTenantID),
+					Type:              infrav1.WorkloadIdentity,
+				},
+			})
+			Expect(err).NotTo(HaveOccurred())
+
 			Log("Waiting for the cluster to be reconciled after moving back to booststrap")
 			clusterResources.Cluster = framework.DiscoveryAndWaitForCluster(ctx, framework.DiscoveryAndWaitForClusterInput{
 				Getter:    input.BootstrapClusterProxy.GetClient(),

test/e2e/data/infrastructure-azure/v1beta1/cluster-template.yaml

@@ -284,10 +284,13 @@ spec:
       - diskSizeGB: 256
         lun: 0
         nameSuffix: etcddisk
+      identity: UserAssigned
       osDisk:
         diskSizeGB: 128
         osType: Linux
       sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""}
+      userAssignedIdentities:
+      - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG:=capz-ci}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY:=cloud-provider-user-identity}
       vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE}
 ---
 apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
@@ -298,10 +301,13 @@ metadata:
 spec:
   template:
     spec:
+      identity: UserAssigned
       osDisk:
         diskSizeGB: 128
         managedDisk:
           storageAccountType: Premium_LRS
         osType: Linux
       sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""}
+      userAssignedIdentities:
+      - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG:=capz-ci}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY:=cloud-provider-user-identity}
       vmSize: ${AZURE_NODE_MACHINE_TYPE}

test/e2e/data/infrastructure-azure/v1beta1/cluster-template/kustomization.yaml

@@ -12,3 +12,5 @@ patches:
 - path: ../../../../../../templates/azure-cluster-identity/azurecluster-identity-ref.yaml
 - path: ../../../../../../templates/test/ci/patches/cluster-label-calico.yaml
 - path: ../../../../../../templates/test/ci/patches/cluster-label-cloud-provider-azure.yaml
+- path: ../../../../../../templates/test/ci/patches/uami-control-plane.yaml
+- path: ../../../../../../templates/test/ci/patches/uami-md-0.yaml