feat: Enforce MD replicas within cluster autoscaler bounds (#1169)

When the current MD `.spec.replicas` is outside of the bounds of the
cluster autoscaler min/max annotations, cluster autoscaler will not
scale the MD up or down. This can occur when either the annotations are
edited or `spec.replicas` is updated.

In a topology-based cluster, CAPI propagates the cluster autoscaler
annotations from
`cluster.spec.topology.workers.machineDeployments[].metadata.annotations`
to the templated `MachineDeployment` resources. If these annotations are
changed so that they do not contain the current number of replicas, then
no scaling up or down will happen to the MachineDeployment by the
cluster autoscaler.

This commit adds a controller that does the following to
MachineDeployments:

- if `spec.replicas` is nil do nothing
- if cluster autoscaler annotations for min or max are missing do
nothing
- if `spec.replicas` is within cluster autoscaler annotations bounds do
nothing
- if `spec.replicas` is not with CA annotations bounds then set
`spec.replicas`
to nil (CAPI MD defaulting webhook will correctly set MD replicas as per

https://github.com/kubernetes-sigs/cluster-api/blob/v1.10.3/internal/webhooks/machinedeployment.go\#L353-L376.

This ensures that scaling up and down by specifying cluster autoscaler
annotations on the machine deployments in
`cluster.spec.topology.workers.machineDeployments[].metadata.annotations`
will always be respected, even if the current number of replicas is
outside the autoscaling specified bounds.

Author: jimmidyson

charts/cluster-api-runtime-extensions-nutanix/README.md

@@ -30,6 +30,7 @@ A Helm chart for cluster-api-runtime-extensions-nutanix
 | certificates.issuer.selfSigned | bool | `true` |  |
 | deployDefaultClusterClasses | bool | `true` |  |
 | deployment.replicas | int | `1` |  |
+| enforceClusterAutoscalerLimits.enabled | bool | `true` |  |
 | env | object | `{}` |  |
 | helmAddonsConfigMap | string | `"default-helm-addons-config"` |  |
 | helmRepository.enabled | bool | `true` |  |

charts/cluster-api-runtime-extensions-nutanix/templates/deployment.yaml

@@ -33,6 +33,7 @@ spec:
         - --namespacesync-enabled={{ .Values.namespaceSync.enabled }}
         - --namespacesync-source-namespace={{ default .Release.Namespace .Values.namespaceSync.sourceNamespace }}
         - --namespacesync-target-namespace-label-key={{ .Values.namespaceSync.targetNamespaceLabelKey }}
+        - --enforce-clusterautoscaler-limits-enabled={{ .Values.enforceClusterAutoscalerLimits.enabled }}
         - --helm-addons-configmap={{ .Values.helmAddonsConfigMap }}
         - --cni.cilium.helm-addon.default-values-template-configmap-name={{ .Values.hooks.cni.cilium.helmAddonStrategy.defaultValueTemplateConfigMap.name }}
         - --nfd.helm-addon.default-values-template-configmap-name={{ .Values.hooks.nfd.helmAddonStrategy.defaultValueTemplateConfigMap.name }}

charts/cluster-api-runtime-extensions-nutanix/templates/role.yaml

@@ -79,6 +79,16 @@ rules:
       - get
       - list
       - watch
+  - apiGroups:
+      - cluster.x-k8s.io
+    resources:
+      - machinedeployments
+    verbs:
+      - get
+      - list
+      - patch
+      - update
+      - watch
   - apiGroups:
       - storage.k8s.io
     resources:

charts/cluster-api-runtime-extensions-nutanix/values.schema.json

@@ -31,6 +31,14 @@
             },
             "type": "object"
         },
+        "enforceClusterAutoscalerLimits": {
+            "properties": {
+                "enabled": {
+                    "type": "boolean"
+                }
+            },
+            "type": "object"
+        },
         "env": {
             "properties": {},
             "type": "object"

charts/cluster-api-runtime-extensions-nutanix/values.yaml

@@ -132,6 +132,16 @@ namespaceSync:
   # By default, sourceNamespace is the helm release namespace.
   sourceNamespace: ""
 
+# Enable the Cluster Autoscaler limits enforcement controller.
+# This controller ensures that the number of replicas in a MachineDeployment
+# does not exceed the limits set by the Cluster Autoscaler annotations.
+# It will also ensure that the number of replicas is at least the minimum
+# number of replicas set by the Cluster Autoscaler annotations.
+# The controller will not enforce the limits if the Cluster Autoscaler annotations
+# are not present on the MachineDeployment.
+enforceClusterAutoscalerLimits:
+  enabled: true
+
 deployment:
   replicas: 1
 

cmd/main.go

@@ -31,6 +31,7 @@ import (
 	caaphv1 "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/external/sigs.k8s.io/cluster-api-addon-provider-helm/api/v1alpha1"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/handlers"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/server"
+	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/controllers/enforceclusterautoscalerlimits"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/controllers/namespacesync"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/feature"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/aws"
@@ -117,6 +118,7 @@ func main() {
 	genericMetaHandlers := generic.New()
 
 	namespacesyncOptions := namespacesync.Options{}
+	enforceClusterAutoscalerLimitsOptions := enforceclusterautoscalerlimits.Options{}
 
 	// Initialize and parse command line flags.
 	logs.AddFlags(pflag.CommandLine, logs.SkipLoggingConfigurationFlags())
@@ -128,6 +130,7 @@ func main() {
 	dockerMetaHandlers.AddFlags(pflag.CommandLine)
 	nutanixMetaHandlers.AddFlags(pflag.CommandLine)
 	namespacesyncOptions.AddFlags(pflag.CommandLine)
+	enforceClusterAutoscalerLimitsOptions.AddFlags(pflag.CommandLine)
 	pflag.CommandLine.SetNormalizeFunc(cliflag.WordSepNormalizeFunc)
 	pflag.CommandLine.AddGoFlagSet(flag.CommandLine)
 
@@ -197,7 +200,6 @@ func main() {
 			SourceClusterClassNamespace: namespacesyncOptions.SourceNamespace,
 			IsTargetNamespace:           namespacesync.NamespaceHasLabelKey(namespacesyncOptions.TargetNamespaceLabelKey),
 		}).SetupWithManager(
-			signalCtx,
 			mgr,
 			&controller.Options{MaxConcurrentReconciles: namespacesyncOptions.Concurrency},
 		); err != nil {
@@ -211,6 +213,23 @@ func main() {
 		}
 	}
 
+	if enforceClusterAutoscalerLimitsOptions.Enabled {
+		if err := (&enforceclusterautoscalerlimits.Reconciler{
+			Client: mgr.GetClient(),
+		}).SetupWithManager(
+			mgr,
+			&controller.Options{MaxConcurrentReconciles: enforceClusterAutoscalerLimitsOptions.Concurrency},
+		); err != nil {
+			setupLog.Error(
+				err,
+				"unable to create controller",
+				"controller",
+				"enforceclusterautoscalerlimits.Reconciler",
+			)
+			os.Exit(1)
+		}
+	}
+
 	mgr.GetWebhookServer().Register("/mutate-v1beta1-cluster", &webhook.Admission{
 		Handler: cluster.NewDefaulter(mgr.GetClient(), admission.NewDecoder(mgr.GetScheme())),
 	})

internal/test/builder/builders.go

@@ -19,6 +19,7 @@ package builder
 
 import (
 	"fmt"
+	"strconv"
 	"strings"
 
 	corev1 "k8s.io/api/core/v1"
@@ -1773,6 +1774,8 @@ type MachineDeploymentBuilder struct {
 	labels                 map[string]string
 	status                 *clusterv1.MachineDeploymentStatus
 	minReadySeconds        *int32
+	minSize                *int32
+	maxSize                *int32
 }
 
 // MachineDeployment creates a MachineDeploymentBuilder with the given name and namespace.
@@ -1831,6 +1834,18 @@ func (m *MachineDeploymentBuilder) WithReplicas(replicas int32) *MachineDeployme
 	return m
 }
 
+// WithMinClusterAutoscalerAnnotation sets the number of CA min annotation for the MachineDeploymentBuilder.
+func (m *MachineDeploymentBuilder) WithMinClusterAutoscalerAnnotation(min int32) *MachineDeploymentBuilder {
+	m.minSize = &min
+	return m
+}
+
+// WithMaxClusterAutoscalerAnnotation sets the number of CA max annotation for the MachineDeploymentBuilder.
+func (m *MachineDeploymentBuilder) WithMaxClusterAutoscalerAnnotation(max int32) *MachineDeploymentBuilder {
+	m.maxSize = &max
+	return m
+}
+
 // WithGeneration sets the passed value on the machine deployments object metadata.
 func (m *MachineDeploymentBuilder) WithGeneration(generation int64) *MachineDeploymentBuilder {
 	m.generation = &generation
@@ -1898,6 +1913,20 @@ func (m *MachineDeploymentBuilder) Build() *clusterv1.MachineDeployment {
 	}
 	obj.Spec.MinReadySeconds = m.minReadySeconds
 
+	if m.minSize != nil {
+		if obj.Annotations == nil {
+			obj.Annotations = map[string]string{}
+		}
+		obj.Annotations[clusterv1.AutoscalerMinSizeAnnotation] = strconv.FormatInt(int64(*m.minSize), 10)
+	}
+
+	if m.maxSize != nil {
+		if obj.Annotations == nil {
+			obj.Annotations = map[string]string{}
+		}
+		obj.Annotations[clusterv1.AutoscalerMaxSizeAnnotation] = strconv.FormatInt(int64(*m.maxSize), 10)
+	}
+
 	return obj
 }
 

pkg/controllers/enforceclusterautoscalerlimits/controller.go

@@ -0,0 +1,148 @@
+// Copyright 2025 Nutanix. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package enforceclusterautoscalerlimits
+
+import (
+	context "context"
+	fmt "fmt"
+	"strconv"
+
+	"github.com/pkg/errors"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
+)
+
+type Reconciler struct {
+	client.Client
+}
+
+func (r *Reconciler) SetupWithManager(
+	mgr ctrl.Manager,
+	options *controller.Options,
+) error {
+	return ctrl.NewControllerManagedBy(mgr).
+		For(&clusterv1.MachineDeployment{}).
+		WithOptions(*options).
+		Complete(r)
+}
+
+func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	logger := ctrl.LoggerFrom(ctx).WithValues("machineDeployment", req.NamespacedName)
+
+	var md clusterv1.MachineDeployment
+	if err := r.Get(ctx, req.NamespacedName, &md); err != nil {
+		if apierrors.IsNotFound(err) {
+			logger.V(5).Info("MachineDeployment not found, skipping reconciliation")
+			return ctrl.Result{}, nil
+		}
+
+		return ctrl.Result{}, fmt.Errorf("failed to get MachineDeployment %s: %w", req.NamespacedName, err)
+	}
+
+	// If replicas is not set, we don't need to do anything.
+	if md.Spec.Replicas == nil {
+		logger.V(5).Info("MachineDeployment has no replicas set, skipping reconciliation")
+		return ctrl.Result{}, nil
+	}
+
+	minReplicas, err := minReplicasFromAnnotations(md.Annotations)
+	if err != nil {
+		// Do nothing if the minSize annotation is missing.
+		if errors.Is(err, errMissingMinAnnotation) {
+			logger.V(5).Info("MachineDeployment has no min size annotation, skipping reconciliation")
+			return ctrl.Result{}, nil
+		}
+
+		return ctrl.Result{}, fmt.Errorf("failed to get min size: %w", err)
+	}
+
+	maxReplicas, err := maxReplicasFromAnnotations(md.Annotations)
+	if err != nil {
+		// Do nothing if the maxSize annotation is missing.
+		if errors.Is(err, errMissingMaxAnnotation) {
+			logger.V(5).Info("MachineDeployment has no max size annotation, skipping reconciliation")
+			return ctrl.Result{}, nil
+		}
+
+		return ctrl.Result{}, fmt.Errorf("failed to get max size: %w", err)
+	}
+
+	if minReplicas > maxReplicas {
+		logger.WithValues("minReplicas", minReplicas, "maxReplicas", maxReplicas).
+			Info("Min replicas is greater than max replicas - skipping reconciliation")
+		return ctrl.Result{}, nil
+	}
+
+	// If the current replicas are within the bounds, do nothing.
+	if int(*md.Spec.Replicas) >= minReplicas && int(*md.Spec.Replicas) <= maxReplicas {
+		return ctrl.Result{}, nil
+	}
+
+	// Otherwise set replicas to nil and depend on CAPI MachineDeployment defaulting to handle
+	// the scaling correctly.
+	// See https://github.com/kubernetes-sigs/cluster-api/blob/v1.10.3/internal/webhooks/machinedeployment.go#L365
+	// for more details.
+	md.Spec.Replicas = nil
+
+	if err := r.Update(ctx, &md); err != nil {
+		return ctrl.Result{}, fmt.Errorf("failed to update MachineDeployment %s: %w", req.NamespacedName, err)
+	}
+
+	return ctrl.Result{}, nil
+}
+
+var (
+	// errMissingMinAnnotation is the error returned when a
+	// machine set does not have an annotation keyed by
+	// nodeGroupMinSizeAnnotationKey.
+	errMissingMinAnnotation = errors.New("missing min annotation")
+
+	// errMissingMaxAnnotation is the error returned when a
+	// machine set does not have an annotation keyed by
+	// nodeGroupMaxSizeAnnotationKey.
+	errMissingMaxAnnotation = errors.New("missing max annotation")
+
+	// errInvalidMinAnnotationValue is the error returned when a
+	// machine set has a non-integral min annotation value.
+	errInvalidMinAnnotation = errors.New("invalid min annotation")
+
+	// errInvalidMaxAnnotationValue is the error returned when a
+	// machine set has a non-integral max annotation value.
+	errInvalidMaxAnnotation = errors.New("invalid max annotation")
+)
+
+// minReplicasFromAnnotations returns the minimum value encoded in the annotations keyed
+// by "cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size".
+// Returns errMissingMinAnnotation if the annotation doesn't exist or
+// errInvalidMinAnnotation if the value is not of type int.
+func minReplicasFromAnnotations(annotations map[string]string) (int, error) {
+	val, found := annotations[clusterv1.AutoscalerMinSizeAnnotation]
+	if !found {
+		return 0, errMissingMinAnnotation
+	}
+	i, err := strconv.Atoi(val)
+	if err != nil {
+		return 0, fmt.Errorf("%w: %v", errInvalidMinAnnotation, err)
+	}
+	return i, nil
+}
+
+// maxReplicasFromAnnotations returns the maximum value encoded in the annotations keyed
+// by "cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size".
+// Returns errMissingMaxAnnotation if the annotation doesn't exist or
+// errInvalidMaxAnnotation if the value is not of type int.
+func maxReplicasFromAnnotations(annotations map[string]string) (int, error) {
+	val, found := annotations[clusterv1.AutoscalerMaxSizeAnnotation]
+	if !found {
+		return 0, errMissingMaxAnnotation
+	}
+	i, err := strconv.Atoi(val)
+	if err != nil {
+		return 0, fmt.Errorf("%w: %v", errInvalidMaxAnnotation, err)
+	}
+	return i, nil
+}