Skip to content

Commit 4320d90

Browse files
nojnhuhnojnhuh
committed
ASOAPI: aggregate agentPoolProfiles for ManagedCluster create
1 parent abba169 commit 4320d90

File tree

9 files changed

+489
-38
lines changed

9 files changed

+489
-38
lines changed

‎exp/controllers/azureasomanagedcontrolplane_controller.go‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,12 @@ func (r *AzureASOManagedControlPlaneReconciler) SetupWithManager(ctx context.Con
7373
infracontroller.ClusterPauseChangeAndInfrastructureReady(log),
7474
),
7575
).
76+
// User errors that CAPZ passes through agentPoolProfiles on create must be fixed in the
77+
// AzureASOManagedMachinePool, so trigger a reconciliation to consume those fixes.
78+
Watches(
79+
&infrav1exp.AzureASOManagedMachinePool{},
80+
handler.EnqueueRequestsFromMapFunc(r.azureASOManagedMachinePoolToAzureASOManagedControlPlane),
81+
).
7682
Owns(&corev1.Secret{}).
7783
Build(r)
7884
if err != nil {
@@ -106,6 +112,19 @@ func clusterToAzureASOManagedControlPlane(_ context.Context, o client.Object) []
106112
return nil
107113
}
108114

115+
func (r *AzureASOManagedControlPlaneReconciler) azureASOManagedMachinePoolToAzureASOManagedControlPlane(ctx context.Context, o client.Object) []ctrl.Request {
116+
asoManagedMachinePool := o.(*infrav1exp.AzureASOManagedMachinePool)
117+
clusterName := asoManagedMachinePool.Labels[clusterv1.ClusterNameLabel]
118+
if clusterName == "" {
119+
return nil
120+
}
121+
cluster, err := util.GetClusterByName(ctx, r.Client, asoManagedMachinePool.Namespace, clusterName)
122+
if client.IgnoreNotFound(err) != nil || cluster == nil {
123+
return nil
124+
}
125+
return clusterToAzureASOManagedControlPlane(ctx, cluster)
126+
}
127+
109128
//+kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=azureasomanagedcontrolplanes,verbs=get;list;watch;create;update;patch;delete
110129
//+kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=azureasomanagedcontrolplanes/status,verbs=get;update;patch
111130
//+kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=azureasomanagedcontrolplanes/finalizers,verbs=update
@@ -180,7 +199,7 @@ func (r *AzureASOManagedControlPlaneReconciler) reconcileNormal(ctx context.Cont
180199
return ctrl.Result{Requeue: true}, nil
181200
}
182201

183-
resources, err := mutators.ApplyMutators(ctx, asoManagedControlPlane.Spec.Resources, mutators.SetManagedClusterDefaults(asoManagedControlPlane, cluster))
202+
resources, err := mutators.ApplyMutators(ctx, asoManagedControlPlane.Spec.Resources, mutators.SetManagedClusterDefaults(r.Client, asoManagedControlPlane, cluster))
184203
if err != nil {
185204
return ctrl.Result{}, err
186205
}

‎exp/mutators/azureasomanagedcontrolplane.go‎

Lines changed: 184 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,32 +18,40 @@ package mutators
1818

1919
import (
2020
"context"
21+
"errors"
2122
"fmt"
2223
"strings"
2324

2425
asocontainerservicev1 "github.com/Azure/azure-service-operator/v2/api/containerservice/v1api20231001"
26+
asocontainerservicev1hub "github.com/Azure/azure-service-operator/v2/api/containerservice/v1api20231001/storage"
2527
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
28+
"sigs.k8s.io/cluster-api-provider-azure/azure"
2629
infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1alpha1"
2730
"sigs.k8s.io/cluster-api-provider-azure/util/tele"
2831
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
32+
"sigs.k8s.io/controller-runtime/pkg/client"
33+
"sigs.k8s.io/controller-runtime/pkg/conversion"
2934
"sigs.k8s.io/controller-runtime/pkg/reconcile"
3035
)
3136

3237
var (
3338
// ErrNoManagedClusterDefined describes an AzureASOManagedControlPlane without a ManagedCluster.
34-
ErrNoManagedClusterDefined = fmt.Errorf("no %s ManagedCluster defined in AzureASOManagedControlPlane spec.resources", asocontainerservicev1.GroupVersion.Group)
39+
ErrNoManagedClusterDefined = fmt.Errorf("no %s ManagedCluster defined in AzureASOManagedControlPlane spec.resources", asocontainerservicev1hub.GroupVersion.Group)
40+
41+
// ErrNoAzureASOManagedMachinePools means no AzureASOManagedMachinePools exist for an AzureASOManagedControlPlane.
42+
ErrNoAzureASOManagedMachinePools = errors.New("no AzureASOManagedMachinePools found for AzureASOManagedControlPlane")
3543
)
3644

3745
// SetManagedClusterDefaults propagates values defined by Cluster API to an ASO ManagedCluster.
38-
func SetManagedClusterDefaults(asoManagedControlPlane *infrav1exp.AzureASOManagedControlPlane, cluster *clusterv1.Cluster) ResourcesMutator {
46+
func SetManagedClusterDefaults(ctrlClient client.Client, asoManagedControlPlane *infrav1exp.AzureASOManagedControlPlane, cluster *clusterv1.Cluster) ResourcesMutator {
3947
return func(ctx context.Context, us []*unstructured.Unstructured) error {
4048
ctx, _, done := tele.StartSpanWithLogger(ctx, "mutators.SetManagedClusterDefaults")
4149
defer done()
4250

4351
var managedCluster *unstructured.Unstructured
4452
var managedClusterPath string
4553
for i, u := range us {
46-
if u.GroupVersionKind().Group == asocontainerservicev1.GroupVersion.Group &&
54+
if u.GroupVersionKind().Group == asocontainerservicev1hub.GroupVersion.Group &&
4755
u.GroupVersionKind().Kind == "ManagedCluster" {
4856
managedCluster = u
4957
managedClusterPath = fmt.Sprintf("spec.resources[%d]", i)
@@ -66,6 +74,10 @@ func SetManagedClusterDefaults(asoManagedControlPlane *infrav1exp.AzureASOManage
6674
return err
6775
}
6876

77+
if err := setManagedClusterAgentPoolProfiles(ctx, ctrlClient, asoManagedControlPlane.Namespace, cluster, managedClusterPath, managedCluster); err != nil {
78+
return err
79+
}
80+
6981
return nil
7082
}
7183
}
@@ -165,3 +177,172 @@ func setManagedClusterPodCIDR(ctx context.Context, cluster *clusterv1.Cluster, m
165177
logMutation(log, setPodCIDR)
166178
return unstructured.SetNestedField(managedCluster.UnstructuredContent(), capiCIDR, podCIDRPath...)
167179
}
180+
181+
func setManagedClusterAgentPoolProfiles(ctx context.Context, ctrlClient client.Client, namespace string, cluster *clusterv1.Cluster, managedClusterPath string, managedCluster *unstructured.Unstructured) error {
182+
ctx, log, done := tele.StartSpanWithLogger(ctx, "mutators.setManagedClusterAgentPoolProfiles")
183+
defer done()
184+
185+
agentPoolProfilesPath := []string{"spec", "agentPoolProfiles"}
186+
userAgentPoolProfiles, agentPoolProfilesFound, err := unstructured.NestedSlice(managedCluster.UnstructuredContent(), agentPoolProfilesPath...)
187+
if err != nil {
188+
return err
189+
}
190+
setAgentPoolProfiles := mutation{
191+
location: managedClusterPath + "." + strings.Join(agentPoolProfilesPath, "."),
192+
val: "nil",
193+
reason: "because agent pool definitions must be inherited from AzureASOManagedMachinePools",
194+
}
195+
if agentPoolProfilesFound {
196+
return Incompatible{
197+
mutation: setAgentPoolProfiles,
198+
userVal: fmt.Sprintf("<slice of length %d>", len(userAgentPoolProfiles)),
199+
}
200+
}
201+
202+
// AKS requires ManagedClusters to be created with agent pools: https://github.com/Azure/azure-service-operator/issues/2791
203+
getMC := &asocontainerservicev1.ManagedCluster{}
204+
err = ctrlClient.Get(ctx, client.ObjectKey{Namespace: namespace, Name: managedCluster.GetName()}, getMC)
205+
if client.IgnoreNotFound(err) != nil {
206+
return err
207+
}
208+
if len(getMC.Status.AgentPoolProfiles) != 0 {
209+
return nil
210+
}
211+
212+
log.V(4).Info("gathering agent pool profiles to include in ManagedCluster create")
213+
agentPools, err := agentPoolsFromManagedMachinePools(ctx, ctrlClient, cluster.Name, namespace)
214+
if err != nil {
215+
return err
216+
}
217+
mc, err := ctrlClient.Scheme().New(managedCluster.GroupVersionKind())
218+
if err != nil {
219+
return err
220+
}
221+
err = ctrlClient.Scheme().Convert(managedCluster, mc, nil)
222+
if err != nil {
223+
return err
224+
}
225+
setAgentPoolProfiles.val = fmt.Sprintf("<slice of length %d>", len(agentPools))
226+
logMutation(log, setAgentPoolProfiles)
227+
err = setAgentPoolProfilesFromAgentPools(mc.(conversion.Convertible), agentPools)
228+
if err != nil {
229+
return err
230+
}
231+
err = ctrlClient.Scheme().Convert(mc, managedCluster, nil)
232+
if err != nil {
233+
return err
234+
}
235+
236+
return nil
237+
}
238+
239+
func agentPoolsFromManagedMachinePools(ctx context.Context, ctrlClient client.Client, clusterName string, namespace string) ([]conversion.Convertible, error) {
240+
ctx, _, done := tele.StartSpanWithLogger(ctx, "mutators.agentPoolsFromManagedMachinePools")
241+
defer done()
242+
243+
asoManagedMachinePools := &infrav1exp.AzureASOManagedMachinePoolList{}
244+
err := ctrlClient.List(ctx, asoManagedMachinePools,
245+
client.InNamespace(namespace),
246+
client.MatchingLabels{
247+
clusterv1.ClusterNameLabel: clusterName,
248+
},
249+
)
250+
if err != nil {
251+
return nil, fmt.Errorf("failed to list AzureASOManagedMachinePools: %w", err)
252+
}
253+
254+
var agentPools []conversion.Convertible
255+
for _, asoManagedMachinePool := range asoManagedMachinePools.Items {
256+
resources, err := ApplyMutators(ctx, asoManagedMachinePool.Spec.Resources)
257+
if err != nil {
258+
return nil, err
259+
}
260+
261+
for _, u := range resources {
262+
if u.GroupVersionKind().Group != asocontainerservicev1hub.GroupVersion.Group ||
263+
u.GroupVersionKind().Kind != "ManagedClustersAgentPool" {
264+
continue
265+
}
266+
267+
agentPool, err := ctrlClient.Scheme().New(u.GroupVersionKind())
268+
if err != nil {
269+
return nil, fmt.Errorf("error creating new %v: %w", u.GroupVersionKind(), err)
270+
}
271+
err = ctrlClient.Scheme().Convert(u, agentPool, nil)
272+
if err != nil {
273+
return nil, err
274+
}
275+
276+
agentPools = append(agentPools, agentPool.(conversion.Convertible))
277+
break
278+
}
279+
}
280+
281+
return agentPools, nil
282+
}
283+
284+
func setAgentPoolProfilesFromAgentPools(managedCluster conversion.Convertible, agentPools []conversion.Convertible) error {
285+
hubMC := &asocontainerservicev1hub.ManagedCluster{}
286+
err := managedCluster.ConvertTo(hubMC)
287+
if err != nil {
288+
return err
289+
}
290+
hubMC.Spec.AgentPoolProfiles = nil
291+
292+
for _, agentPool := range agentPools {
293+
hubPool := &asocontainerservicev1hub.ManagedClustersAgentPool{}
294+
err := agentPool.ConvertTo(hubPool)
295+
if err != nil {
296+
return err
297+
}
298+
299+
profile := asocontainerservicev1hub.ManagedClusterAgentPoolProfile{
300+
AvailabilityZones: hubPool.Spec.AvailabilityZones,
301+
CapacityReservationGroupReference: hubPool.Spec.CapacityReservationGroupReference,
302+
Count: hubPool.Spec.Count,
303+
CreationData: hubPool.Spec.CreationData,
304+
EnableAutoScaling: hubPool.Spec.EnableAutoScaling,
305+
EnableEncryptionAtHost: hubPool.Spec.EnableEncryptionAtHost,
306+
EnableFIPS: hubPool.Spec.EnableFIPS,
307+
EnableNodePublicIP: hubPool.Spec.EnableNodePublicIP,
308+
EnableUltraSSD: hubPool.Spec.EnableUltraSSD,
309+
GpuInstanceProfile: hubPool.Spec.GpuInstanceProfile,
310+
HostGroupReference: hubPool.Spec.HostGroupReference,
311+
KubeletConfig: hubPool.Spec.KubeletConfig,
312+
KubeletDiskType: hubPool.Spec.KubeletDiskType,
313+
LinuxOSConfig: hubPool.Spec.LinuxOSConfig,
314+
MaxCount: hubPool.Spec.MaxCount,
315+
MaxPods: hubPool.Spec.MaxPods,
316+
MinCount: hubPool.Spec.MinCount,
317+
Mode: hubPool.Spec.Mode,
318+
Name: azure.AliasOrNil[string](&hubPool.Spec.AzureName),
319+
NetworkProfile: hubPool.Spec.NetworkProfile,
320+
NodeLabels: hubPool.Spec.NodeLabels,
321+
NodePublicIPPrefixReference: hubPool.Spec.NodePublicIPPrefixReference,
322+
NodeTaints: hubPool.Spec.NodeTaints,
323+
OrchestratorVersion: hubPool.Spec.OrchestratorVersion,
324+
OsDiskSizeGB: hubPool.Spec.OsDiskSizeGB,
325+
OsDiskType: hubPool.Spec.OsDiskType,
326+
OsSKU: hubPool.Spec.OsSKU,
327+
OsType: hubPool.Spec.OsType,
328+
PodSubnetReference: hubPool.Spec.PodSubnetReference,
329+
PowerState: hubPool.Spec.PowerState,
330+
PropertyBag: hubPool.Spec.PropertyBag,
331+
ProximityPlacementGroupReference: hubPool.Spec.ProximityPlacementGroupReference,
332+
ScaleDownMode: hubPool.Spec.ScaleDownMode,
333+
ScaleSetEvictionPolicy: hubPool.Spec.ScaleSetEvictionPolicy,
334+
ScaleSetPriority: hubPool.Spec.ScaleSetPriority,
335+
SpotMaxPrice: hubPool.Spec.SpotMaxPrice,
336+
Tags: hubPool.Spec.Tags,
337+
Type: hubPool.Spec.Type,
338+
UpgradeSettings: hubPool.Spec.UpgradeSettings,
339+
VmSize: hubPool.Spec.VmSize,
340+
VnetSubnetReference: hubPool.Spec.VnetSubnetReference,
341+
WorkloadRuntime: hubPool.Spec.WorkloadRuntime,
342+
}
343+
344+
hubMC.Spec.AgentPoolProfiles = append(hubMC.Spec.AgentPoolProfiles, profile)
345+
}
346+
347+
return managedCluster.ConvertFrom(hubMC)
348+
}

0 commit comments

Comments
 (0)