Skip to content

🌱 autoscaling: try to gather the version information from MachineSets and e2e test improvements #5774

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Nov 28, 2025
Merged

🌱 autoscaling: try to gather the version information from MachineSets and e2e test improvements #5774

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions config/rbac/role.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@ rules:
- clusters/status
- machinedeployments
- machines/status
- machinesets
verbs:
- get
- list
Expand Down
71 changes: 66 additions & 5 deletions controllers/awsmachinetemplate_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,12 @@ import (
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/utils/ptr"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/builder"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller"
"sigs.k8s.io/controller-runtime/pkg/event"
"sigs.k8s.io/controller-runtime/pkg/handler"
"sigs.k8s.io/controller-runtime/pkg/predicate"

infrav1 "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2"
ekscontrolplanev1 "sigs.k8s.io/cluster-api-provider-aws/v2/controlplane/eks/api/v1beta2"
Expand Down Expand Up @@ -75,12 +78,23 @@ func (r *AWSMachineTemplateReconciler) SetupWithManager(ctx context.Context, mgr
Watches(
&clusterv1.MachineDeployment{},
handler.EnqueueRequestsFromMapFunc(r.machineDeploymentToAWSMachineTemplate),
// Only emit events for creation to reconcile in case the MachineDeployment got created after the AWSMachineTemplate was reconciled.
builder.WithPredicates(resourceCreatedPredicate),
).
Watches(
&clusterv1.MachineSet{},
handler.EnqueueRequestsFromMapFunc(r.machineSetToAWSMachineTemplate),
// Only emit events for creation to reconcile in case the MachineSet got created after the AWSMachineTemplate was reconciled.
builder.WithPredicates(resourceCreatedPredicate),
)

// Optionally watch KubeadmControlPlane if the CRD exists
// Watch KubeadmControlPlane if they exist.
if _, err := mgr.GetRESTMapper().RESTMapping(schema.GroupKind{Group: controlplanev1.GroupVersion.Group, Kind: "KubeadmControlPlane"}, controlplanev1.GroupVersion.Version); err == nil {
b = b.Watches(&controlplanev1.KubeadmControlPlane{},
handler.EnqueueRequestsFromMapFunc(r.kubeadmControlPlaneToAWSMachineTemplate))
handler.EnqueueRequestsFromMapFunc(r.kubeadmControlPlaneToAWSMachineTemplate),
// Only emit events for creation to reconcile in case the KubeadmControlPlane got created after the AWSMachineTemplate was reconciled.
builder.WithPredicates(resourceCreatedPredicate),
)
}

_, err := b.Build(r)
Expand All @@ -95,7 +109,7 @@ func (r *AWSMachineTemplateReconciler) SetupWithManager(ctx context.Context, mgr
// +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=awsmachinetemplates/status,verbs=get;update;patch
// +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=awsclusters,verbs=get;list;watch
// +kubebuilder:rbac:groups=cluster.x-k8s.io,resources=clusters,verbs=get;list;watch
// +kubebuilder:rbac:groups=cluster.x-k8s.io,resources=machinedeployments,verbs=get;list;watch
// +kubebuilder:rbac:groups=cluster.x-k8s.io,resources=machinedeployments;machinesets,verbs=get;list;watch
// +kubebuilder:rbac:groups="",resources=events,verbs=get;list;watch;create;update;patch

// Reconcile populates capacity information for AWSMachineTemplate.
Expand Down Expand Up @@ -387,7 +401,22 @@ func (r *AWSMachineTemplateReconciler) getKubernetesVersion(ctx context.Context,
return "", errors.Wrap(err, "failed to get parent list options")
}

// Try to find version from MachineDeployment first
// Try to find version from MachineSet first
machineSetList := &clusterv1.MachineSetList{}
if err := r.List(ctx, machineSetList, listOpts...); err != nil {
return "", errors.Wrap(err, "failed to list MachineSets")
}

// Find MachineSets that reference this AWSMachineTemplate
for _, ms := range machineSetList.Items {
if ms.Spec.Template.Spec.InfrastructureRef.Kind == awsMachineTemplateKind &&
ms.Spec.Template.Spec.InfrastructureRef.Name == template.Name &&
ms.Spec.Template.Spec.Version != "" {
return ms.Spec.Template.Spec.Version, nil
}
}

// If not found, try MachineDeployment.
machineDeploymentList := &clusterv1.MachineDeploymentList{}
if err := r.List(ctx, machineDeploymentList, listOpts...); err != nil {
return "", errors.Wrap(err, "failed to list MachineDeployments")
Expand All @@ -402,7 +431,7 @@ func (r *AWSMachineTemplateReconciler) getKubernetesVersion(ctx context.Context,
}
}

// If not found in MachineDeployment, try KubeadmControlPlane
// If not found, try KubeadmControlPlane
kcpList := &controlplanev1.KubeadmControlPlaneList{}
if err := r.List(ctx, kcpList, listOpts...); err != nil {
return "", errors.Wrap(err, "failed to list KubeadmControlPlanes")
Expand Down Expand Up @@ -492,3 +521,35 @@ func (r *AWSMachineTemplateReconciler) machineDeploymentToAWSMachineTemplate(ctx
},
}
}

// machineSetToAWSMachineTemplate maps MachineSet to AWSMachineTemplate reconcile requests.
// This enables the controller to reconcile AWSMachineTemplate when its owner MachineSet is created or updated,
// ensuring that nodeInfo can be populated even if the cache hasn't synced yet.
func (r *AWSMachineTemplateReconciler) machineSetToAWSMachineTemplate(ctx context.Context, o client.Object) []ctrl.Request {
md, ok := o.(*clusterv1.MachineSet)
if !ok {
return nil
}

// Check if it references an AWSMachineTemplate
if md.Spec.Template.Spec.InfrastructureRef.Kind != awsMachineTemplateKind {
return nil
}

// Return reconcile request for the referenced AWSMachineTemplate
return []ctrl.Request{
{
NamespacedName: client.ObjectKey{
Namespace: md.Namespace,
Name: md.Spec.Template.Spec.InfrastructureRef.Name,
},
},
}
}
Comment on lines +528 to +548
Copy link
Contributor

@LiangquanLi930 LiangquanLi930 Nov 28, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more. 

func (r *AWSMachineTemplateReconciler) kubeadmControlPlaneToAWSMachineTemplate(ctx context.Context, o client.Object) []ctrl.Request
func (r *AWSMachineTemplateReconciler) machineDeploymentToAWSMachineTemplate(ctx context.Context, o client.Object) []ctrl.Request
func (r *AWSMachineTemplateReconciler) machineSetToAWSMachineTemplate(ctx context.Context, o client.Object) []ctrl.Request

These three functions are very similar. Should we refactor them a bit, similar to how we handle objectToAWSMachineTemplate?

Copy link
Member

@damdo damdo Nov 28, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'd prefer to prioritize merging this if it comes back green, in the interest of getting the release out. But let's see what @chrischdi says

Copy link
Member Author

@chrischdi chrischdi Nov 28, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Imho not worth the complexity compared to these simple functions.


var resourceCreatedPredicate = predicate.Funcs{
CreateFunc: func(e event.CreateEvent) bool { return true },
UpdateFunc: func(e event.UpdateEvent) bool { return false },
DeleteFunc: func(e event.DeleteEvent) bool { return false },
GenericFunc: func(e event.GenericEvent) bool { return true },
}
32 changes: 6 additions & 26 deletions test/e2e/suites/unmanaged/unmanaged_functional_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -354,7 +354,7 @@ var _ = ginkgo.Context("[unmanaged] [functional]", func() {
Expect(len(workerMachines)).To(Equal(1))
Expect(len(controlPlaneMachines)).To(Equal(1))

ginkgo.By("Verifying AWSMachineTemplate capacity is populated for autoscaling from zero")
ginkgo.By("Verifying AWSMachineTemplate capacity and nodeInfo is populated for autoscaling from zero")
Eventually(func(g Gomega) {
awsMachineTemplateList := &infrav1.AWSMachineTemplateList{}
g.Expect(e2eCtx.Environment.BootstrapClusterProxy.GetClient().List(ctx, awsMachineTemplateList, client.InNamespace(namespace.Name))).To(Succeed())
Expand All @@ -364,32 +364,12 @@ var _ = ginkgo.Context("[unmanaged] [functional]", func() {
capacity := template.Status.Capacity
_, hasCPU := capacity[corev1.ResourceCPU]
_, hasMemory := capacity[corev1.ResourceMemory]
if hasCPU && hasMemory {
ginkgo.By(fmt.Sprintf("AWSMachineTemplate %s has capacity populated: %v", template.Name, capacity))
return
}
g.Expect(hasCPU).To(BeTrue(), "Expected AWSMachineTemplate %s to have .status.capacity for CPU set", template.Name)
g.Expect(hasMemory).To(BeTrue(), "Expected AWSMachineTemplate %s to have .status.capacity for memory set", template.Name)
g.Expect(template.Status.NodeInfo).ToNot(BeNil(), "Expected AWSMachineTemplate %s to have .status.nodeInfo set", template.Name)
g.Expect(template.Status.NodeInfo.Architecture).ToNot(BeEmpty(), "Expected AWSMachineTemplate %s to have .status.nodeInfo.architecture set", template.Name)
g.Expect(template.Status.NodeInfo.OperatingSystem).ToNot(BeEmpty(), "Expected AWSMachineTemplate %s to have .status.nodeInfo.operatingSystem set", template.Name)
}
g.Expect(false).To(BeTrue(), "Expected at least one AWSMachineTemplate to have capacity with CPU and Memory")
}, e2eCtx.E2EConfig.GetIntervals(specName, "wait-deployment")...).Should(Succeed())

ginkgo.By("Verifying AWSMachineTemplate nodeInfo is populated")
Eventually(func(g Gomega) {
awsMachineTemplateList := &infrav1.AWSMachineTemplateList{}
g.Expect(e2eCtx.Environment.BootstrapClusterProxy.GetClient().List(ctx, awsMachineTemplateList, client.InNamespace(namespace.Name))).To(Succeed())
g.Expect(awsMachineTemplateList.Items).ToNot(BeEmpty())

for _, template := range awsMachineTemplateList.Items {
nodeInfo := template.Status.NodeInfo
if nodeInfo == nil {
continue
}
arch := string(nodeInfo.Architecture)
if (arch == "amd64" || arch == "arm64") && nodeInfo.OperatingSystem != "" {
ginkgo.By(fmt.Sprintf("AWSMachineTemplate %s has nodeInfo populated: %v", template.Name, nodeInfo))
return
}
}
g.Expect(false).To(BeTrue(), "Expected at least one AWSMachineTemplate to have valid nodeInfo")
}, e2eCtx.E2EConfig.GetIntervals(specName, "wait-deployment")...).Should(Succeed())
})
})
Expand Down