ROX-28223: Pause-reconcile handler (#426)

* ROX-28223: Pause-reconcile annotation

Co-authored-by: Marcin Owsiany <[email protected]>

* Remove Stackrox specific workaround

* Pause reconcile handler

* Fix panic & tests

* Code review suggestions

* Fix lint issues

Signed-off-by: Per Goncalves da Silva <[email protected]>

---------

Signed-off-by: Per Goncalves da Silva <[email protected]>
Co-authored-by: Marcin Owsiany <[email protected]>
Co-authored-by: Per Goncalves da Silva <[email protected]>

Author: 3 people

pkg/reconciler/internal/conditions/conditions.go

@@ -29,10 +29,12 @@ const (
 	TypeDeployed       = "Deployed"
 	TypeReleaseFailed  = "ReleaseFailed"
 	TypeIrreconcilable = "Irreconcilable"
+	TypePaused         = "Paused"
 
-	ReasonInstallSuccessful   = status.ConditionReason("InstallSuccessful")
-	ReasonUpgradeSuccessful   = status.ConditionReason("UpgradeSuccessful")
-	ReasonUninstallSuccessful = status.ConditionReason("UninstallSuccessful")
+	ReasonInstallSuccessful            = status.ConditionReason("InstallSuccessful")
+	ReasonUpgradeSuccessful            = status.ConditionReason("UpgradeSuccessful")
+	ReasonUninstallSuccessful          = status.ConditionReason("UninstallSuccessful")
+	ReasonPauseReconcileAnnotationTrue = status.ConditionReason("PauseReconcileAnnotationTrue")
 
 	ReasonErrorGettingClient       = status.ConditionReason("ErrorGettingClient")
 	ReasonErrorGettingValues       = status.ConditionReason("ErrorGettingValues")
@@ -59,6 +61,10 @@ func Irreconcilable(stat corev1.ConditionStatus, reason status.ConditionReason,
 	return newCondition(TypeIrreconcilable, stat, reason, message)
 }
 
+func Paused(stat corev1.ConditionStatus, reason status.ConditionReason, message interface{}) status.Condition {
+	return newCondition(TypePaused, stat, reason, message)
+}
+
 func newCondition(t status.ConditionType, s corev1.ConditionStatus, r status.ConditionReason, m interface{}) status.Condition {
 	message := fmt.Sprintf("%s", m)
 	return status.Condition{

pkg/reconciler/reconciler.go

@@ -83,6 +83,7 @@ type Reconciler struct {
 	maxReleaseHistory                *int
 	skipPrimaryGVKSchemeRegistration bool
 	controllerSetupFuncs             []ControllerSetupFunc
+	pauseHandler                     PauseReconcileHandlerFunc
 
 	annotSetupOnce       sync.Once
 	annotations          map[string]struct{}
@@ -439,6 +440,33 @@ func WithUninstallAnnotations(as ...annotation.Uninstall) Option {
 	}
 }
 
+// PauseReconcileHandlerFunc defines a function type that determines whether reconciliation should be paused
+// for a given custom resource
+type PauseReconcileHandlerFunc func(ctx context.Context, obj *unstructured.Unstructured) (bool, error)
+
+// WithPauseReconcileHandler is an Option that sets a PauseReconcile handler, which is a function that
+// determines whether reconciliation should be paused for the custom resource watched by this reconciler.
+//
+// Example usage: WithPauseReconcileHandler(PauseReconcileIfAnnotationTrue("my.domain/pause-reconcile"))
+func WithPauseReconcileHandler(handler PauseReconcileHandlerFunc) Option {
+	return func(r *Reconciler) error {
+		r.pauseHandler = handler
+		return nil
+	}
+}
+
+// PauseReconcileIfAnnotationTrue returns a PauseReconcileHandlerFunc that pauses reconciliation if the given
+// annotation is present and set to "true"
+func PauseReconcileIfAnnotationTrue(annotationName string) PauseReconcileHandlerFunc {
+	return func(_ context.Context, obj *unstructured.Unstructured) (bool, error) {
+		if v, ok := obj.GetAnnotations()[annotationName]; ok && v == "true" {
+			return true, nil
+		}
+
+		return false, nil
+	}
+}
+
 // WithPreHook is an Option that configures the reconciler to run the given
 // PreHook just before performing any actions (e.g. install, upgrade, uninstall,
 // or reconciliation).
@@ -591,6 +619,28 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Re
 		}
 	}()
 
+	if r.pauseHandler != nil {
+		paused, err := r.pauseHandler(ctx, obj)
+		if err != nil {
+			log.Error(err, "pause reconcile handler failed")
+		}
+
+		if paused {
+			log.Info("Reconcile is paused for this resource.")
+			u.UpdateStatus(
+				updater.EnsureCondition(conditions.Paused(corev1.ConditionTrue, conditions.ReasonPauseReconcileAnnotationTrue, "")),
+				updater.EnsureConditionUnknown(conditions.TypeIrreconcilable),
+				updater.EnsureConditionUnknown(conditions.TypeDeployed),
+				updater.EnsureConditionUnknown(conditions.TypeInitialized),
+				updater.EnsureConditionUnknown(conditions.TypeReleaseFailed),
+				updater.EnsureDeployedRelease(nil),
+			)
+			return ctrl.Result{}, nil
+		}
+	}
+
+	u.UpdateStatus(updater.EnsureCondition(conditions.Paused(corev1.ConditionFalse, "", "")))
+
 	actionClient, err := r.actionClientGetter.ActionClientFor(ctx, obj)
 	if err != nil {
 		u.UpdateStatus(

pkg/reconciler/reconciler_test.go

@@ -1419,6 +1419,69 @@ var _ = Describe("Reconciler", func() {
 									verifyNoRelease(ctx, mgr.GetClient(), obj.GetNamespace(), obj.GetName(), currentRelease)
 								})
 
+								By("ensuring the finalizer is removed and the CR is deleted", func() {
+									err := mgr.GetAPIReader().Get(ctx, objKey, obj)
+									Expect(apierrors.IsNotFound(err)).To(BeTrue())
+								})
+							})
+						})
+						When("pause-reconcile annotation is present", func() {
+							It("pauses reconciliation", func() {
+								By("adding a pause-reconcile handler to the Reconciler", func() {
+									pauseHandler := WithPauseReconcileHandler(PauseReconcileIfAnnotationTrue("my.domain/pause-reconcile"))
+									Expect(pauseHandler(r)).To(Succeed())
+								})
+
+								By("adding the pause-reconcile annotation to the CR", func() {
+									Expect(mgr.GetClient().Get(ctx, objKey, obj)).To(Succeed())
+									obj.SetAnnotations(map[string]string{"my.domain/pause-reconcile": "true"})
+									obj.Object["spec"] = map[string]interface{}{"replicaCount": "666"}
+									Expect(mgr.GetClient().Update(ctx, obj)).To(Succeed())
+								})
+
+								By("deleting the CR", func() {
+									Expect(mgr.GetClient().Delete(ctx, obj)).To(Succeed())
+								})
+
+								By("successfully reconciling a request when paused", func() {
+									res, err := r.Reconcile(ctx, req)
+									Expect(res).To(Equal(reconcile.Result{}))
+									Expect(err).ToNot(HaveOccurred())
+								})
+
+								By("getting the CR", func() {
+									Expect(mgr.GetAPIReader().Get(ctx, objKey, obj)).To(Succeed())
+								})
+
+								By("verifying the CR status is Paused", func() {
+									objStat := &objStatus{}
+									Expect(runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, objStat)).To(Succeed())
+									Expect(objStat.Status.Conditions.IsTrueFor(conditions.TypePaused)).To(BeTrue())
+								})
+
+								By("verifying the release has not changed", func() {
+									rel, err := ac.Get(obj.GetName())
+									Expect(err).ToNot(HaveOccurred())
+									Expect(rel).NotTo(BeNil())
+									Expect(*rel).To(Equal(*currentRelease))
+								})
+
+								By("removing the pause-reconcile annotation from the CR", func() {
+									Expect(mgr.GetClient().Get(ctx, objKey, obj)).To(Succeed())
+									obj.SetAnnotations(nil)
+									Expect(mgr.GetClient().Update(ctx, obj)).To(Succeed())
+								})
+
+								By("successfully reconciling a request", func() {
+									res, err := r.Reconcile(ctx, req)
+									Expect(res).To(Equal(reconcile.Result{}))
+									Expect(err).ToNot(HaveOccurred())
+								})
+
+								By("verifying the release is uninstalled", func() {
+									verifyNoRelease(ctx, mgr.GetClient(), obj.GetNamespace(), obj.GetName(), currentRelease)
+								})
+
 								By("ensuring the finalizer is removed and the CR is deleted", func() {
 									err := mgr.GetAPIReader().Get(ctx, objKey, obj)
 									Expect(apierrors.IsNotFound(err)).To(BeTrue())