multi tenancy support for azure managed control plane

Author: shysank

azure/scope/identity.go

@@ -25,6 +25,7 @@ import (
 	"github.com/Azure/go-autorest/autorest/adal"
 	"github.com/pkg/errors"
 	infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha4"
+	infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1alpha4"
 	"sigs.k8s.io/cluster-api-provider-azure/util/identity"
 	"sigs.k8s.io/cluster-api-provider-azure/util/system"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha4"
@@ -54,7 +55,14 @@ type AzureClusterCredentialsProvider struct {
 	AzureCluster *infrav1.AzureCluster
 }
 
+// ManagedControlPlaneCredentialsProvider wraps AzureCredentialsProvider with AzureManagedControlPlane.
+type ManagedControlPlaneCredentialsProvider struct {
+	AzureCredentialsProvider
+	AzureManagedControlPlane *infrav1exp.AzureManagedControlPlane
+}
+
 var _ CredentialsProvider = (*AzureClusterCredentialsProvider)(nil)
+var _ CredentialsProvider = (*ManagedControlPlaneCredentialsProvider)(nil)
 
 // NewAzureClusterCredentialsProvider creates a new AzureClusterCredentialsProvider from the supplied inputs.
 func NewAzureClusterCredentialsProvider(ctx context.Context, kubeClient client.Client, azureCluster *infrav1.AzureCluster) (*AzureClusterCredentialsProvider, error) {
@@ -92,6 +100,42 @@ func (p *AzureClusterCredentialsProvider) GetAuthorizer(ctx context.Context, res
 	return p.AzureCredentialsProvider.GetAuthorizer(ctx, resourceManagerEndpoint, p.AzureCluster.ObjectMeta)
 }
 
+// NewManagedControlPlaneCredentialsProvider creates a new ManagedControlPlaneCredentialsProvider from the supplied inputs.
+func NewManagedControlPlaneCredentialsProvider(ctx context.Context, kubeClient client.Client, managedControlPlane *infrav1exp.AzureManagedControlPlane) (*ManagedControlPlaneCredentialsProvider, error) {
+	if managedControlPlane.Spec.IdentityRef == nil {
+		return nil, errors.New("failed to generate new ManagedControlPlaneCredentialsProvider from empty identityName")
+	}
+
+	ref := managedControlPlane.Spec.IdentityRef
+	// if the namespace isn't specified then assume it's in the same namespace as the AzureManagedControlPlane
+	namespace := ref.Namespace
+	if namespace == "" {
+		namespace = managedControlPlane.Namespace
+	}
+	identity := &infrav1.AzureClusterIdentity{}
+	key := client.ObjectKey{Name: ref.Name, Namespace: namespace}
+	if err := kubeClient.Get(ctx, key, identity); err != nil {
+		return nil, errors.Errorf("failed to retrieve AzureClusterIdentity external object %q/%q: %v", key.Namespace, key.Name, err)
+	}
+
+	if identity.Spec.Type != infrav1.ServicePrincipal {
+		return nil, errors.New("AzureClusterIdentity is not of type Service Principal")
+	}
+
+	return &ManagedControlPlaneCredentialsProvider{
+		AzureCredentialsProvider{
+			Client:   kubeClient,
+			Identity: identity,
+		},
+		managedControlPlane,
+	}, nil
+}
+
+// GetAuthorizer returns an Azure authorizer based on the provided azure identity. It delegates to AzureCredentialsProvider with AzureManagedControlPlane metadata.
+func (p *ManagedControlPlaneCredentialsProvider) GetAuthorizer(ctx context.Context, resourceManagerEndpoint string) (autorest.Authorizer, error) {
+	return p.AzureCredentialsProvider.GetAuthorizer(ctx, resourceManagerEndpoint, p.AzureManagedControlPlane.ObjectMeta)
+}
+
 // GetAuthorizer returns an Azure authorizer based on the provided azure identity and cluster metadata.
 func (p *AzureCredentialsProvider) GetAuthorizer(ctx context.Context, resourceManagerEndpoint string, clusterMeta metav1.ObjectMeta) (autorest.Authorizer, error) {
 	azureIdentityType, err := getAzureIdentityType(p.Identity)

azure/scope/managedcontrolplane.go

@@ -48,7 +48,7 @@ type ManagedControlPlaneScopeParams struct {
 
 // NewManagedControlPlaneScope creates a new Scope from the supplied parameters.
 // This is meant to be called for each reconcile iteration.
-func NewManagedControlPlaneScope(params ManagedControlPlaneScopeParams) (*ManagedControlPlaneScope, error) {
+func NewManagedControlPlaneScope(ctx context.Context, params ManagedControlPlaneScopeParams) (*ManagedControlPlaneScope, error) {
 	if params.Cluster == nil {
 		return nil, errors.New("failed to generate new scope from nil Cluster")
 	}
@@ -61,8 +61,19 @@ func NewManagedControlPlaneScope(params ManagedControlPlaneScopeParams) (*Manage
 		params.Logger = klogr.New()
 	}
 
-	if err := params.AzureClients.setCredentials(params.ControlPlane.Spec.SubscriptionID, ""); err != nil {
-		return nil, errors.Wrap(err, "failed to create Azure session")
+	if params.ControlPlane.Spec.IdentityRef == nil {
+		if err := params.AzureClients.setCredentials(params.ControlPlane.Spec.SubscriptionID, ""); err != nil {
+			return nil, errors.Wrap(err, "failed to create Azure session")
+		}
+	} else {
+		credentialsProvider, err := NewManagedControlPlaneCredentialsProvider(ctx, params.Client, params.ControlPlane)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to init credentials provider")
+		}
+
+		if err := params.AzureClients.setCredentialsWithProvider(ctx, params.ControlPlane.Spec.SubscriptionID, "", credentialsProvider); err != nil {
+			return nil, errors.Wrap(err, "failed to configure azure settings and credentials for Identity")
+		}
 	}
 
 	helper, err := patch.NewHelper(params.PatchTarget, params.Client)

config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml

@@ -192,6 +192,31 @@ spec:
               dnsServiceIP:
                 description: DNSServiceIP is an IP address assigned to the Kubernetes DNS service. It must be within the Kubernetes service address range specified in serviceCidr.
                 type: string
+              identityRef:
+                description: IdentityRef is a reference to a AzureClusterIdentity to be used when reconciling this cluster
+                properties:
+                  apiVersion:
+                    description: API version of the referent.
+                    type: string
+                  fieldPath:
+                    description: 'If referring to a piece of an object instead of an entire object, this string should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. For example, if the object reference is to a container within a pod, this would take on a value like: "spec.containers{name}" (where "name" refers to the name of the container that triggered the event) or if no container name is specified "spec.containers[2]" (container with index 2 in this pod). This syntax is chosen only to have some well-defined way of referencing a part of an object. TODO: this design is not final and this field is subject to change in the future.'
+                    type: string
+                  kind:
+                    description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+                    type: string
+                  name:
+                    description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+                    type: string
+                  namespace:
+                    description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/'
+                    type: string
+                  resourceVersion:
+                    description: 'Specific resourceVersion to which this reference is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency'
+                    type: string
+                  uid:
+                    description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids'
+                    type: string
+                type: object
               loadBalancerSKU:
                 description: LoadBalancerSKU is the SKU of the loadBalancer to be provisioned.
                 enum:

exp/api/v1alpha3/azuremanagedcontrolplane_conversion.go

@@ -17,6 +17,7 @@ limitations under the License.
 package v1alpha3
 
 import (
+	apiconversion "k8s.io/apimachinery/pkg/conversion"
 	expv1alpha4 "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1alpha4"
 	utilconversion "sigs.k8s.io/cluster-api/util/conversion"
 	"sigs.k8s.io/controller-runtime/pkg/conversion"
@@ -36,6 +37,8 @@ func (src *AzureManagedControlPlane) ConvertTo(dstRaw conversion.Hub) error { //
 		return err
 	}
 
+	dst.Spec.IdentityRef = restored.Spec.IdentityRef
+
 	return nil
 }
 
@@ -54,3 +57,8 @@ func (dst *AzureManagedControlPlane) ConvertFrom(srcRaw conversion.Hub) error {
 
 	return nil
 }
+
+// Convert_v1alpha4_AzureManagedControlPlaneSpec_To_v1alpha3_AzureManagedControlPlaneSpec is an autogenerated conversion function.
+func Convert_v1alpha4_AzureManagedControlPlaneSpec_To_v1alpha3_AzureManagedControlPlaneSpec(in *expv1alpha4.AzureManagedControlPlaneSpec, out *AzureManagedControlPlaneSpec, s apiconversion.Scope) error {
+	return autoConvert_v1alpha4_AzureManagedControlPlaneSpec_To_v1alpha3_AzureManagedControlPlaneSpec(in, out, s)
+}

exp/api/v1alpha3/zz_generated.conversion.go

@@ -82,6 +82,10 @@ type AzureManagedControlPlaneSpec struct {
 	// +kubebuilder:validation:Enum=Basic;Standard
 	// +optional
 	LoadBalancerSKU *string `json:"loadBalancerSKU,omitempty"`
+
+	// IdentityRef is a reference to a AzureClusterIdentity to be used when reconciling this cluster
+	// +optional
+	IdentityRef *corev1.ObjectReference `json:"identityRef,omitempty"`
 }
 
 // ManagedControlPlaneVirtualNetwork describes a virtual network required to provision AKS clusters.

exp/api/v1alpha4/azuremanagedcontrolplane_types.go

@@ -20,6 +20,8 @@ import (
 	"context"
 	"time"
 
+	"sigs.k8s.io/cluster-api/util/patch"
+
 	"github.com/go-logr/logr"
 	"github.com/pkg/errors"
 	"go.opentelemetry.io/otel/attribute"
@@ -150,7 +152,7 @@ func (r *AzureManagedControlPlaneReconciler) Reconcile(ctx context.Context, req
 	// needs to happen before trying to fetch the default pool to avoid circular deletion dependencies
 	if !azureControlPlane.DeletionTimestamp.IsZero() {
 		// Create the scope.
-		mcpScope, err := scope.NewManagedControlPlaneScope(scope.ManagedControlPlaneScopeParams{
+		mcpScope, err := scope.NewManagedControlPlaneScope(ctx, scope.ManagedControlPlaneScopeParams{
 			Client:       r.Client,
 			Logger:       log,
 			Cluster:      cluster,
@@ -192,8 +194,29 @@ func (r *AzureManagedControlPlaneReconciler) Reconcile(ctx context.Context, req
 
 	log = log.WithValues("machinePool", ownerPool.Name)
 
+	// check if the control plane's namespace is allowed for this identity and update owner references for the identity.
+	if azureControlPlane.Spec.IdentityRef != nil {
+		identity, err := infracontroller.GetClusterIdentityFromRef(ctx, r.Client, azureControlPlane.Namespace, azureControlPlane.Spec.IdentityRef)
+		if err != nil {
+			return reconcile.Result{}, err
+		}
+		if !scope.IsClusterNamespaceAllowed(ctx, r.Client, identity.Spec.AllowedNamespaces, azureControlPlane.Namespace) {
+			return reconcile.Result{}, errors.New("AzureClusterIdentity list of allowed namespaces doesn't include current azure managed control plane namespace")
+		}
+		if identity.Namespace == azureControlPlane.Namespace {
+			patchHelper, err := patch.NewHelper(identity, r.Client)
+			if err != nil {
+				return reconcile.Result{}, errors.Wrap(err, "failed to init patch helper")
+			}
+			identity.ObjectMeta.OwnerReferences = azureControlPlane.GetOwnerReferences()
+			if err := patchHelper.Patch(ctx, identity); err != nil {
+				return reconcile.Result{}, err
+			}
+		}
+	}
+
 	// Create the scope.
-	mcpScope, err := scope.NewManagedControlPlaneScope(scope.ManagedControlPlaneScopeParams{
+	mcpScope, err := scope.NewManagedControlPlaneScope(ctx, scope.ManagedControlPlaneScopeParams{
 		Client:           r.Client,
 		Logger:           log,
 		Cluster:          cluster,

exp/api/v1alpha4/zz_generated.deepcopy.go

@@ -191,7 +191,7 @@ func (r *AzureManagedMachinePoolReconciler) Reconcile(ctx context.Context, req c
 	}
 
 	// Create the scope.
-	mcpScope, err := scope.NewManagedControlPlaneScope(scope.ManagedControlPlaneScopeParams{
+	mcpScope, err := scope.NewManagedControlPlaneScope(ctx, scope.ManagedControlPlaneScopeParams{
 		Client:           r.Client,
 		Logger:           log,
 		ControlPlane:     controlPlane,