✨Add an option to control initial rollout of HelmReleaseProxies

api/v1alpha1/condition_consts.go

@@ -42,9 +42,20 @@ const (
 	// ClusterSelectionFailedReason indicates that the HelmChartProxy controller failed to select the workload Clusters.
 	ClusterSelectionFailedReason = "ClusterSelectionFailed"
 
+	// HelmReleaseProxiesRolloutNotCompleteReason indicates that the initial rollout
+	// of HelmReleaseProxies has not been completed.
+	HelmReleaseProxiesRolloutNotCompleteReason = "HelmReleaseProxiesRolloutNotComplete"
+
+	// HelmReleaseProxiesRolloutUndefinedReason indicates that HelmChartProxy doesn't
+	// use Rollout Step Size to reconcile HelmReleaseProxies.
+	HelmReleaseProxiesRolloutUndefinedReason = "HelmReleaseProxiesRolloutUndefined"
+
 	// HelmReleaseProxiesReadyCondition indicates that the HelmReleaseProxies are ready, meaning that the Helm installation, upgrade
 	// or deletion is complete.
 	HelmReleaseProxiesReadyCondition clusterv1.ConditionType = "HelmReleaseProxiesReady"
+
+	// HelmReleaseProxiesRolloutCompletedCondition indicates if the initial rollout of HelmReleaseProxies is complete.
+	HelmReleaseProxiesRolloutCompletedCondition clusterv1.ConditionType = "HelmReleaseProxiesRolloutCompleted"
 )
 
 // HelmReleaseProxy Conditions and Reasons.

api/v1alpha1/helmchartproxy_types.go

@@ -19,6 +19,7 @@ package v1alpha1
 import (
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 )
 
@@ -84,6 +85,15 @@ type HelmChartProxySpec struct {
 	// +optional
 	ReconcileStrategy string `json:"reconcileStrategy,omitempty"`
 
+	// RolloutStepSize is an opt-in feature that defines a step size during the
+	// initial rollout of HelmReleaseProxies on matching clusters. Once all
+	// rolled out HelmReleaseProxy resources are ready=true, the next batch of
+	// HelmReleaseProxy resources are reconciled. If undefined, will default to
+	// creating HelmReleaseProxy resources for all matching clusters.
+	// e.g. an int (5) or percentage of count of total matching clusters (25%)
+	// +optional
+	RolloutStepSize *intstr.IntOrString `json:"rolloutStepSize,omitempty"`
+
 	// Options represents CLI flags passed to Helm operations (i.e. install, upgrade, delete) and
 	// include options such as wait, skipCRDs, timeout, waitForJobs, etc.
 	// +optional

config/crd/bases/addons.cluster.x-k8s.io_helmchartproxies.yaml

@@ -279,6 +279,18 @@ spec:
                   RepoURL is the URL of the Helm chart repository.
                   e.g. chart-path oci://repo-url/chart-name as repoURL: oci://repo-url and https://repo-url/chart-name as repoURL: https://repo-url
                 type: string
+              rolloutStepSize:
+                anyOf:
+                - type: integer
+                - type: string
+                description: |-
+                  RolloutStepSize is an opt-in feature that defines a step size during the
+                  initial rollout of HelmReleaseProxies on matching clusters. Once all
+                  rolled out HelmReleaseProxy resources are ready=true, the next batch of
+                  HelmReleaseProxy resources are reconciled. If undefined, will default to
+                  creating HelmReleaseProxy resources for all matching clusters.
+                  e.g. an int (5) or percentage of count of total matching clusters (25%)
+                x-kubernetes-int-or-string: true
               tlsConfig:
                 description: TLSConfig contains the TLS configuration for a HelmChartProxy.
                 properties:

controllers/helmchartproxy/helmchartproxy_controller.go

@@ -18,12 +18,15 @@ package helmchartproxy
 
 import (
 	"context"
+	"slices"
 
 	"github.com/pkg/errors"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/intstr"
 	addonsv1alpha1 "sigs.k8s.io/cluster-api-addon-provider-helm/api/v1alpha1"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	"sigs.k8s.io/cluster-api/util/conditions"
@@ -45,6 +48,18 @@ type HelmChartProxyReconciler struct {
 	WatchFilterValue string
 }
 
+// helmReleaseProxyRolloutMeta is used to gather HelmReleaseProxy  rollout
+// metadata for matching clusters.
+type helmReleaseProxyRolloutMeta struct {
+	cluster clusterv1.Cluster
+
+	// Identifies whether HelmReleaseProxy exists for the cluster.
+	hrpExists bool
+
+	// Identifies whether HelmReleaseProxy's ready condition is True.
+	hrpReady bool
+}
+
 // SetupWithManager sets up the controller with the Manager.
 func (r *HelmChartProxyReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, options controller.Options) error {
 	log := ctrl.LoggerFrom(ctx)
@@ -197,12 +212,161 @@ func (r *HelmChartProxyReconciler) reconcileNormal(ctx context.Context, helmChar
 		}
 	}
 
-	for _, cluster := range clusters {
-		// Don't reconcile if the Cluster is being deleted
-		if !cluster.DeletionTimestamp.IsZero() {
-			continue
+	// RolloutStepSize is defined; check if count of HelmReleaseProxies matches
+	// the count of matching clusters.
+	if helmChartProxy.Spec.RolloutStepSize != nil {
+		if len(clusters) != len(helmReleaseProxies) {
+			// Set HelmReleaseProxiesRolloutCompletedCondition to false as
+			// HelmReleaseProxies are being rolled out.
+			conditions.MarkFalse(
+				helmChartProxy,
+				addonsv1alpha1.HelmReleaseProxiesRolloutCompletedCondition,
+				addonsv1alpha1.HelmReleaseProxiesRolloutNotCompleteReason,
+				clusterv1.ConditionSeverityInfo,
+				"%d Helm release proxies not yet rolled out",
+				len(clusters)-len(helmReleaseProxies),
+			)
+
+			// Identifies clusters by their NamespacedName and gathers their
+			// helmReleaseProxyRolloutMeta.
+			clusterNnRolloutMeta := map[string]*helmReleaseProxyRolloutMeta{}
+			for _, c := range clusters {
+				nn := getNamespacedNameStringFor(c.Namespace, c.Name)
+				clusterNnRolloutMeta[nn] = &helmReleaseProxyRolloutMeta{
+					cluster: c,
+				}
+			}
+			for _, h := range helmReleaseProxies {
+				ref := h.Spec.ClusterRef
+				nn := getNamespacedNameStringFor(ref.Namespace, ref.Name)
+				meta := clusterNnRolloutMeta[nn]
+				meta.hrpExists = true
+				meta.hrpReady = conditions.IsTrue(&h, addonsv1alpha1.HelmReleaseReadyCondition)
+			}
+
+			// Sort helmReleaseProxy rollout metadata by cluster namespaced name to
+			// ensure orderliness.
+			rolloutMetaSorted := make([]*helmReleaseProxyRolloutMeta, len(clusterNnRolloutMeta))
+			i := 0
+			for _, m := range clusterNnRolloutMeta {
+				rolloutMetaSorted[i] = m
+				i++
+			}
+			for m := range clusterNnRolloutMeta {
+				delete(clusterNnRolloutMeta, m)
+			}
+
+			slices.SortStableFunc(rolloutMetaSorted, func(a, b *helmReleaseProxyRolloutMeta) int {
+				nnA := getNamespacedNameStringFor(a.cluster.Namespace, a.cluster.Name)
+				nnB := getNamespacedNameStringFor(b.cluster.Namespace, b.cluster.Name)
+				if nnA < nnB {
+					return -1
+				}
+
+				if nnA > nnB {
+					return 1
+				}
+
+				return 0
+			})
+
+			// If HelmReleaseProxiesReadyCondition is Unknown, create the first batch
+			// of HelmReleaseProxies and exit.
+			if conditions.IsUnknown(helmChartProxy, addonsv1alpha1.HelmReleaseProxiesReadyCondition) {
+				count := 0
+				stepSize, err := intstr.GetScaledValueFromIntOrPercent(helmChartProxy.Spec.RolloutStepSize, len(clusters), true)
+				if err != nil {
+					return err
+				}
+
+				// If HelmReleaseProxiesReadyCondition is Unknown and the first batch of HelmReleaseProxies have
+				// been created, then exit early.
+				if stepSize == len(helmReleaseProxies) {
+					return nil
+				}
+
+				for _, meta := range rolloutMetaSorted {
+					// The first batch of helmReleaseProxies have been reconciled.
+					if count >= stepSize {
+						return nil
+					}
+
+					err := r.reconcileForCluster(ctx, helmChartProxy, meta.cluster)
+					if err != nil {
+						return err
+					}
+					count++
+				}
+				// In cases where the count of remaining HelmReleaseProxies to be rolled
+				// out is less than rollout step size.
+				return nil
+			}
+
+			// If HelmReleaseProxiesReadyCondition is false, reconcile existing
+			// HelmReleaseProxies and exit.
+			if conditions.IsFalse(helmChartProxy, addonsv1alpha1.HelmReleaseProxiesReadyCondition) {
+				for _, meta := range rolloutMetaSorted {
+					if meta.hrpExists {
+						err := r.reconcileForCluster(ctx, helmChartProxy, meta.cluster)
+						if err != nil {
+							return err
+						}
+					}
+				}
+
+				return nil
+			}
+
+			// HelmReleaseProxyReadyCondition is True; continue with reconciling the
+			// next batch of HelmReleaseProxies.
+			count := 0
+			stepSize, err := intstr.GetScaledValueFromIntOrPercent(helmChartProxy.Spec.RolloutStepSize, len(clusters), true)
+			if err != nil {
+				return err
+			}
+
+			for _, meta := range rolloutMetaSorted {
+				// Exit if HelmReleaseProxyReadyCondition has not caught up to existing
+				// HelmReleaseProxies status.
+				if meta.hrpExists && !meta.hrpReady {
+					return nil
+				}
+
+				// The next batch of helmReleaseProxies have been reconciled.
+				if count >= stepSize {
+					return nil
+				}
+
+				// Skip reconciling the cluster if its HelmReleaseProxy already exists.
+				if meta.hrpExists {
+					continue
+				}
+				err := r.reconcileForCluster(ctx, helmChartProxy, meta.cluster)
+				if err != nil {
+					return err
+				}
+				count++
+			}
+			// In cases where the count of remaining HelmReleaseProxies to be rolled
+			// out is less than rollout step size.
+			return nil
+		} else {
+			// RolloutStepSize is defined and all HelmReleaseProxies have been rolled out.
+			conditions.MarkTrue(helmChartProxy, addonsv1alpha1.HelmReleaseProxiesRolloutCompletedCondition)
 		}
+	} else {
+		// RolloutStepSize is undefined. Set HelmReleaseProxiesRolloutCompletedCondition to True with reason.
+		conditions.MarkTrueWithNegativePolarity(
+			helmChartProxy,
+			addonsv1alpha1.HelmReleaseProxiesRolloutCompletedCondition,
+			addonsv1alpha1.HelmReleaseProxiesRolloutUndefinedReason,
+			clusterv1.ConditionSeverityInfo,
+			"HelmChartProxy does not use rollout step",
+		)
+	}
 
+	// Continue with reconciling for all clusters after initial rollout.
+	for _, cluster := range clusters {
 		err := r.reconcileForCluster(ctx, helmChartProxy, cluster)
 		if err != nil {
 			return err
@@ -321,6 +485,7 @@ func patchHelmChartProxy(ctx context.Context, patchHelper *patch.Helper, helmCha
 		conditions.WithConditions(
 			addonsv1alpha1.HelmReleaseProxySpecsUpToDateCondition,
 			addonsv1alpha1.HelmReleaseProxiesReadyCondition,
+			addonsv1alpha1.HelmReleaseProxiesRolloutCompletedCondition,
 		),
 	)
 
@@ -332,6 +497,7 @@ func patchHelmChartProxy(ctx context.Context, patchHelper *patch.Helper, helmCha
 			clusterv1.ReadyCondition,
 			addonsv1alpha1.HelmReleaseProxySpecsUpToDateCondition,
 			addonsv1alpha1.HelmReleaseProxiesReadyCondition,
+			addonsv1alpha1.HelmReleaseProxiesRolloutCompletedCondition,
 		}},
 		patch.WithStatusObservedGeneration{},
 	)
@@ -407,3 +573,8 @@ func HelmReleaseProxyToHelmChartProxyMapper(ctx context.Context, o client.Object
 
 	return nil
 }
+
+// getNamespacedNameStringFor to retrieve the namespaced name as a string.
+func getNamespacedNameStringFor(namespace, name string) string {
+	return types.NamespacedName{Namespace: namespace, Name: name}.String()
+}

controllers/helmchartproxy/helmchartproxy_controller_phases.go

@@ -57,6 +57,11 @@ func (r *HelmChartProxyReconciler) deleteOrphanedHelmReleaseProxies(ctx context.
 
 // reconcileForCluster will create or update a HelmReleaseProxy for the given cluster.
 func (r *HelmChartProxyReconciler) reconcileForCluster(ctx context.Context, helmChartProxy *addonsv1alpha1.HelmChartProxy, cluster clusterv1.Cluster) error {
+	// Don't reconcile if the Cluster is being deleted
+	if !cluster.DeletionTimestamp.IsZero() {
+		return nil
+	}
+
 	log := ctrl.LoggerFrom(ctx)
 
 	// Don't reconcile if the Cluster or the helmChartProxy is paused.