[v0.14] - Fixes race condition when BD secrets are transiently unavailable (#4726)

* Fix BundleDeployents race condition

---------

Signed-off-by: Xavi Garcia <xavi.garcia@suse.com>

Author: 0xavi0

charts/fleet-crd/templates/crds.yaml

@@ -962,6 +962,18 @@ spec:
                     the bundle.
                   nullable: true
                   type: string
+                waitingForValues:
+                  description: 'WaitingForValues is set to true by the bundle controller
+                    when the
+
+                    options secret for this BundleDeployment could not be found (e.g.
+
+                    due to transient API server pressure). While true, the agent skips
+
+                    reconciliation to avoid deploying with missing Helm values. The
+
+                    controller clears this flag once the secret is successfully loaded.'
+                  type: boolean
               type: object
             status:
               properties:

internal/cmd/agent/controller/bundledeployment_controller.go

@@ -141,6 +141,10 @@ func (r *BundleDeploymentReconciler) Reconcile(ctx context.Context, req ctrl.Req
 
 		return ctrl.Result{}, err
 	}
+	if bd.Spec.WaitingForValues {
+		logger.V(1).Info("BundleDeployment waiting for options secret to become available, skipping deployment")
+		return ctrl.Result{}, nil
+	}
 
 	// load the bundledeployment options from the secret, if present
 	if bd.Spec.ValuesHash != "" {

internal/cmd/cli/target.go

@@ -101,7 +101,7 @@ func (t *Target) Run(cmd *cobra.Command, args []string) error {
 	}
 
 	builder := target.New(client, client)
-	matchedTargets, err := builder.Targets(ctx, bundle, manifestID)
+	matchedTargets, _, err := builder.Targets(ctx, bundle, manifestID)
 	if err != nil {
 		return err
 	}

internal/cmd/controller/reconciler/bundle_controller.go

@@ -25,6 +25,7 @@ import (
 	"github.com/rancher/fleet/internal/metrics"
 	"github.com/rancher/fleet/internal/ocistorage"
 	fleet "github.com/rancher/fleet/pkg/apis/fleet.cattle.io/v1alpha1"
+	"github.com/rancher/fleet/pkg/durations"
 	fleetevent "github.com/rancher/fleet/pkg/event"
 	"github.com/rancher/fleet/pkg/sharding"
 	corev1 "k8s.io/api/core/v1"
@@ -62,7 +63,7 @@ type Store interface {
 }
 
 type TargetBuilder interface {
-	Targets(ctx context.Context, bundle *fleet.Bundle, manifestID string) ([]*target.Target, error)
+	Targets(ctx context.Context, bundle *fleet.Bundle, manifestID string) ([]*target.Target, bool, error)
 }
 
 // BundleReconciler reconciles a Bundle object
@@ -243,7 +244,7 @@ func (r *BundleReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
 		}
 	}
 
-	matchedTargets, err := r.Builder.Targets(ctx, bundle, manifestID)
+	matchedTargets, secretsMissing, err := r.Builder.Targets(ctx, bundle, manifestID)
 	if err != nil {
 		return ctrl.Result{},
 			r.updateErrorStatus(
@@ -328,13 +329,20 @@ func (r *BundleReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
 		bd.Spec.OCIContents = contentsInOCI
 		bd.Spec.HelmChartOptions = bundle.Spec.HelmOpOptions
 
-		valuesHash, optionsSecret, err := r.manageOptionsSecret(ctx, bd)
-		if err != nil {
-			return r.computeResult(ctx, logger, bundleOrig, bundle, "failed to initialize options secret", err)
-		}
+		var optionsSecret *corev1.Secret
+		// If the options secret was unavailable during Targets(). Preserve the
+		// existing ValuesHash (already present in bd.Spec from BundleDeployment())
+		// and skip writing the secret so we don't overwrite it with empty values.
+		if !bd.Spec.WaitingForValues {
+			var valuesHash string
+			valuesHash, optionsSecret, err = r.manageOptionsSecret(ctx, bd)
+			if err != nil {
+				return r.computeResult(ctx, logger, bundleOrig, bundle, "failed to initialize options secret", err)
+			}
 
-		// Changes in the values hash trigger a bundle deployment reconcile.
-		bd.Spec.ValuesHash = valuesHash
+			// Changes in the values hash trigger a bundle deployment reconcile.
+			bd.Spec.ValuesHash = valuesHash
+		}
 
 		// When content resources are stored in etcd, we need to keep track of the content resource so they
 		// are properly gargabe-collected by the content controller.
@@ -402,6 +410,13 @@ func (r *BundleReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
 		return ctrl.Result{}, errutil.NewAggregate(merr)
 	}
 
+	if secretsMissing {
+		// At least one BundleDeployment had its options secret transiently
+		// unavailable. Requeue so we can retry loading those values; we cannot
+		// rely on an external event to re-trigger the reconcile.
+		return ctrl.Result{RequeueAfter: durations.DefaultRequeueAfter}, errutil.NewAggregate(merr)
+	}
+
 	return ctrl.Result{}, errutil.NewAggregate(merr)
 }