Merge pull request #4777 from nojnhuh/v2-mpnodes

ASOAPI: reconcile nodes for AzureASOManagedMachinePool

Author: k8s-ci-robot

config/crd/bases/infrastructure.cluster.x-k8s.io_azureasomanagedmachinepools.yaml

@@ -41,6 +41,13 @@ spec:
             description: AzureASOManagedMachinePoolSpec defines the desired state
               of AzureASOManagedMachinePool.
             properties:
+              providerIDList:
+                description: |-
+                  ProviderIDList is the list of cloud provider IDs for the instances. It fulfills Cluster API's machine
+                  pool infrastructure provider contract.
+                items:
+                  type: string
+                type: array
               resources:
                 description: Resources are embedded ASO resources to be managed by
                   this resource.
@@ -53,6 +60,12 @@ spec:
             description: AzureASOManagedMachinePoolStatus defines the observed state
               of AzureASOManagedMachinePool.
             properties:
+              replicas:
+                description: |-
+                  Replicas is the current number of provisioned replicas. It fulfills Cluster API's machine pool
+                  infrastructure provider contract.
+                format: int32
+                type: integer
               resources:
                 items:
                   description: ResourceStatus represents the status of a resource.

exp/api/v1alpha1/azureasomanagedmachinepool_types.go

@@ -30,6 +30,11 @@ type AzureASOManagedMachinePoolSpec struct {
 
 // AzureASOManagedMachinePoolStatus defines the observed state of AzureASOManagedMachinePool.
 type AzureASOManagedMachinePoolStatus struct {
+	// Replicas is the current number of provisioned replicas. It fulfills Cluster API's machine pool
+	// infrastructure provider contract.
+	//+optional
+	Replicas int32 `json:"replicas"`
+
 	//+optional
 	Resources []ResourceStatus `json:"resources,omitempty"`
 }

exp/api/v1alpha1/azureasomanagedmachinepooltemplate_types.go

@@ -33,6 +33,10 @@ type AzureASOManagedMachinePoolResource struct {
 
 // AzureASOManagedMachinePoolTemplateResourceSpec defines the desired state of the templated resource.
 type AzureASOManagedMachinePoolTemplateResourceSpec struct {
+	// ProviderIDList is the list of cloud provider IDs for the instances. It fulfills Cluster API's machine
+	// pool infrastructure provider contract.
+	ProviderIDList []string `json:"providerIDList,omitempty"`
+
 	// Resources are embedded ASO resources to be managed by this resource.
 	//+optional
 	Resources []runtime.RawExtension `json:"resources,omitempty"`

exp/api/v1alpha1/zz_generated.deepcopy.go

@@ -19,8 +19,13 @@ package controllers
 import (
 	"context"
 	"fmt"
+	"slices"
 
+	asocontainerservicev1 "github.com/Azure/azure-service-operator/v2/api/containerservice/v1api20231001"
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/utils/ptr"
 	infracontroller "sigs.k8s.io/cluster-api-provider-azure/controllers"
 	infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1alpha1"
 	"sigs.k8s.io/cluster-api-provider-azure/util/tele"
@@ -44,10 +49,16 @@ import (
 type AzureASOManagedMachinePoolReconciler struct {
 	client.Client
 	WatchFilterValue string
+	Tracker          ClusterTracker
 
 	newResourceReconciler func(*infrav1exp.AzureASOManagedMachinePool, []*unstructured.Unstructured) resourceReconciler
 }
 
+// ClusterTracker wraps a CAPI remote.ClusterCacheTracker.
+type ClusterTracker interface {
+	GetClient(context.Context, types.NamespacedName) (client.Client, error)
+}
+
 // SetupWithManager sets up the controller with the Manager.
 func (r *AzureASOManagedMachinePoolReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager) error {
 	_, log, done := tele.StartSpanWithLogger(ctx,
@@ -184,7 +195,6 @@ func (r *AzureASOManagedMachinePoolReconciler) Reconcile(ctx context.Context, re
 
 //nolint:unparam // these parameters will be used soon enough
 func (r *AzureASOManagedMachinePoolReconciler) reconcileNormal(ctx context.Context, asoManagedMachinePool *infrav1exp.AzureASOManagedMachinePool, machinePool *expv1.MachinePool, cluster *clusterv1.Cluster) (ctrl.Result, error) {
-	//nolint:all // ctx will be used soon
 	ctx, log, done := tele.StartSpanWithLogger(ctx,
 		"controllers.AzureASOManagedMachinePoolReconciler.reconcileNormal",
 	)
@@ -200,6 +210,19 @@ func (r *AzureASOManagedMachinePoolReconciler) reconcileNormal(ctx context.Conte
 	if err != nil {
 		return ctrl.Result{}, err
 	}
+
+	var agentPoolName string
+	for _, resource := range us {
+		if resource.GroupVersionKind().Group == asocontainerservicev1.GroupVersion.Group &&
+			resource.GroupVersionKind().Kind == "ManagedClustersAgentPool" {
+			agentPoolName = resource.GetName()
+			break
+		}
+	}
+	if agentPoolName == "" {
+		return ctrl.Result{}, reconcile.TerminalError(fmt.Errorf("no %s ManagedClustersAgentPools defined in AzureASOManagedMachinePool spec.resources", asocontainerservicev1.GroupVersion.Group))
+	}
+
 	resourceReconciler := r.newResourceReconciler(asoManagedMachinePool, us)
 	err = resourceReconciler.Reconcile(ctx)
 	if err != nil {
@@ -211,9 +234,56 @@ func (r *AzureASOManagedMachinePoolReconciler) reconcileNormal(ctx context.Conte
 		}
 	}
 
+	agentPool := &asocontainerservicev1.ManagedClustersAgentPool{}
+	err = r.Get(ctx, client.ObjectKey{Namespace: asoManagedMachinePool.Namespace, Name: agentPoolName}, agentPool)
+	if err != nil {
+		return ctrl.Result{}, fmt.Errorf("error getting ManagedClustersAgentPool: %w", err)
+	}
+
+	managedCluster := &asocontainerservicev1.ManagedCluster{}
+	err = r.Get(ctx, client.ObjectKey{Namespace: agentPool.Namespace, Name: agentPool.Owner().Name}, managedCluster)
+	if err != nil {
+		return ctrl.Result{}, fmt.Errorf("error getting ManagedCluster: %w", err)
+	}
+	if managedCluster.Status.NodeResourceGroup == nil {
+		return ctrl.Result{}, nil
+	}
+	rg := *managedCluster.Status.NodeResourceGroup
+
+	clusterClient, err := r.Tracker.GetClient(ctx, util.ObjectKey(cluster))
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+	nodes := &corev1.NodeList{}
+	err = clusterClient.List(ctx, nodes,
+		client.MatchingLabels(expectedNodeLabels(agentPool.AzureName(), rg)),
+	)
+	if err != nil {
+		return ctrl.Result{}, fmt.Errorf("failed to list nodes in workload cluster: %w", err)
+	}
+	providerIDs := make([]string, 0, len(nodes.Items))
+	for _, node := range nodes.Items {
+		if node.Spec.ProviderID == "" {
+			// the node will receive a provider id soon
+			return ctrl.Result{Requeue: true}, nil
+		}
+		providerIDs = append(providerIDs, node.Spec.ProviderID)
+	}
+	// Prevent a different order from updating the spec.
+	slices.Sort(providerIDs)
+	asoManagedMachinePool.Spec.ProviderIDList = providerIDs
+	asoManagedMachinePool.Status.Replicas = int32(ptr.Deref(agentPool.Status.Count, 0))
+
 	return ctrl.Result{}, nil
 }
 
+func expectedNodeLabels(poolName, nodeRG string) map[string]string {
+	return map[string]string{
+		"kubernetes.azure.com/agentpool": poolName,
+		"kubernetes.azure.com/cluster":   nodeRG,
+	}
+}
+
 //nolint:unparam // these parameters will be used soon enough
 func (r *AzureASOManagedMachinePoolReconciler) reconcilePause(ctx context.Context, asoManagedMachinePool *infrav1exp.AzureASOManagedMachinePool, cluster *clusterv1.Cluster) (ctrl.Result, error) {
 	//nolint:all // ctx will be used soon

exp/controllers/azureasomanagedmachinepool_controller.go

@@ -18,15 +18,19 @@ package controllers
 
 import (
 	"context"
+	"encoding/json"
 	"testing"
 	"time"
 
+	asocontainerservicev1 "github.com/Azure/azure-service-operator/v2/api/containerservice/v1api20231001"
+	"github.com/Azure/azure-service-operator/v2/pkg/genruntime"
 	. "github.com/onsi/gomega"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/utils/ptr"
 	infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1alpha1"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	expv1 "sigs.k8s.io/cluster-api/exp/api/v1beta1"
@@ -35,6 +39,17 @@ import (
 	fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake"
 )
 
+type FakeClusterTracker struct {
+	getClientFunc func(context.Context, types.NamespacedName) (client.Client, error)
+}
+
+func (c *FakeClusterTracker) GetClient(ctx context.Context, name types.NamespacedName) (client.Client, error) {
+	if c.getClientFunc == nil {
+		return nil, nil
+	}
+	return c.getClientFunc(ctx, name)
+}
+
 func TestAzureASOManagedMachinePoolReconcile(t *testing.T) {
 	ctx := context.Background()
 
@@ -43,6 +58,7 @@ func TestAzureASOManagedMachinePoolReconcile(t *testing.T) {
 		infrav1exp.AddToScheme,
 		clusterv1.AddToScheme,
 		expv1.AddToScheme,
+		asocontainerservicev1.AddToScheme,
 	)
 	NewGomegaWithT(t).Expect(sb.AddToScheme(s)).To(Succeed())
 	fakeClientBuilder := func() *fakeclient.ClientBuilder {
@@ -210,6 +226,19 @@ func TestAzureASOManagedMachinePoolReconcile(t *testing.T) {
 					clusterv1.ClusterFinalizer,
 				},
 			},
+			Spec: infrav1exp.AzureASOManagedMachinePoolSpec{
+				AzureASOManagedMachinePoolTemplateResourceSpec: infrav1exp.AzureASOManagedMachinePoolTemplateResourceSpec{
+					Resources: []runtime.RawExtension{
+						{
+							Raw: apJSON(g, &asocontainerservicev1.ManagedClustersAgentPool{
+								ObjectMeta: metav1.ObjectMeta{
+									Name: "ap",
+								},
+							}),
+						},
+					},
+				},
+			},
 		}
 		machinePool := &expv1.MachinePool{
 			ObjectMeta: metav1.ObjectMeta{
@@ -259,6 +288,30 @@ func TestAzureASOManagedMachinePoolReconcile(t *testing.T) {
 				},
 			},
 		}
+		asoManagedCluster := &asocontainerservicev1.ManagedCluster{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "mc",
+				Namespace: cluster.Namespace,
+			},
+			Status: asocontainerservicev1.ManagedCluster_STATUS{
+				NodeResourceGroup: ptr.To("MC_rg"),
+			},
+		}
+		asoAgentPool := &asocontainerservicev1.ManagedClustersAgentPool{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "ap",
+				Namespace: cluster.Namespace,
+			},
+			Spec: asocontainerservicev1.ManagedClusters_AgentPool_Spec{
+				AzureName: "pool1",
+				Owner: &genruntime.KnownResourceReference{
+					Name: asoManagedCluster.Name,
+				},
+			},
+			Status: asocontainerservicev1.ManagedClusters_AgentPool_STATUS{
+				Count: ptr.To(3),
+			},
+		}
 		asoManagedMachinePool := &infrav1exp.AzureASOManagedMachinePool{
 			ObjectMeta: metav1.ObjectMeta{
 				Name:      "ammp",
@@ -274,6 +327,15 @@ func TestAzureASOManagedMachinePoolReconcile(t *testing.T) {
 					clusterv1.ClusterFinalizer,
 				},
 			},
+			Spec: infrav1exp.AzureASOManagedMachinePoolSpec{
+				AzureASOManagedMachinePoolTemplateResourceSpec: infrav1exp.AzureASOManagedMachinePoolTemplateResourceSpec{
+					Resources: []runtime.RawExtension{
+						{
+							Raw: apJSON(g, asoAgentPool),
+						},
+					},
+				},
+			},
 		}
 		machinePool := &expv1.MachinePool{
 			ObjectMeta: metav1.ObjectMeta{
@@ -285,7 +347,7 @@ func TestAzureASOManagedMachinePoolReconcile(t *testing.T) {
 			},
 		}
 		c := fakeClientBuilder().
-			WithObjects(asoManagedMachinePool, machinePool, cluster).
+			WithObjects(asoManagedMachinePool, machinePool, cluster, asoAgentPool, asoManagedCluster).
 			Build()
 		r := &AzureASOManagedMachinePoolReconciler{
 			Client: c,
@@ -296,10 +358,48 @@ func TestAzureASOManagedMachinePoolReconcile(t *testing.T) {
 					},
 				}
 			},
+			Tracker: &FakeClusterTracker{
+				getClientFunc: func(_ context.Context, _ types.NamespacedName) (client.Client, error) {
+					return fakeclient.NewClientBuilder().
+						WithObjects(
+							&corev1.Node{
+								ObjectMeta: metav1.ObjectMeta{
+									Name:   "node1",
+									Labels: expectedNodeLabels(asoAgentPool.AzureName(), *asoManagedCluster.Status.NodeResourceGroup),
+								},
+								Spec: corev1.NodeSpec{
+									ProviderID: "azure://node1",
+								},
+							},
+							&corev1.Node{
+								ObjectMeta: metav1.ObjectMeta{
+									Name:   "node2",
+									Labels: expectedNodeLabels(asoAgentPool.AzureName(), *asoManagedCluster.Status.NodeResourceGroup),
+								},
+								Spec: corev1.NodeSpec{
+									ProviderID: "azure://node2",
+								},
+							},
+							&corev1.Node{
+								ObjectMeta: metav1.ObjectMeta{
+									Name: "no-labels",
+								},
+								Spec: corev1.NodeSpec{
+									ProviderID: "azure://node3",
+								},
+							},
+						).
+						Build(), nil
+				},
+			},
 		}
 		result, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(asoManagedMachinePool)})
 		g.Expect(err).NotTo(HaveOccurred())
 		g.Expect(result).To(Equal(ctrl.Result{}))
+
+		g.Expect(r.Get(ctx, client.ObjectKeyFromObject(asoManagedMachinePool), asoManagedMachinePool)).To(Succeed())
+		g.Expect(asoManagedMachinePool.Spec.ProviderIDList).To(ConsistOf("azure://node1", "azure://node2"))
+		g.Expect(asoManagedMachinePool.Status.Replicas).To(Equal(int32(3)))
 	})
 
 	t.Run("successfully reconciles pause", func(t *testing.T) {
@@ -410,3 +510,10 @@ func TestAzureASOManagedMachinePoolReconcile(t *testing.T) {
 		g.Expect(result).To(Equal(ctrl.Result{}))
 	})
 }
+
+func apJSON(g Gomega, ap *asocontainerservicev1.ManagedClustersAgentPool) []byte {
+	ap.SetGroupVersionKind(asocontainerservicev1.GroupVersion.WithKind("ManagedClustersAgentPool"))
+	j, err := json.Marshal(ap)
+	g.Expect(err).NotTo(HaveOccurred())
+	return j
+}