Allow uninstall mode to be cancelled

Previously, the running container could only move *into* uninstallation
mode - once it was in that mode, it needed to be restarted to restore
normal operations. To improve the reliability of uninstall/reinstall
scenarios, it can now move back into normal operations automatically.

All reconciles that were skipped during uninstallation are now just
delayed 5 minutes. A "normal" uninstallation should be completed within
that amount of time. If the uninstallation was aborted, then this should
ensure no reconciles are permanently missed.

Refs:
 - https://issues.redhat.com/browse/ACM-6941

Signed-off-by: Justin Kulikauskas <[email protected]>

Author: JustinKuli

controllers/gatekeepersync/gatekeeper_constraint_sync.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"strings"
 	"sync"
+	"time"
 
 	depclient "github.com/stolostron/kubernetes-dependency-watches/client"
 	admissionregistration "k8s.io/api/admissionregistration/v1"
@@ -93,7 +94,7 @@ func (r *GatekeeperConstraintReconciler) Reconcile(
 	if uninstall.DeploymentIsUninstalling {
 		log.Info("Skipping reconcile because the deployment is in uninstallation mode")
 
-		return reconcile.Result{}, nil
+		return reconcile.Result{RequeueAfter: 5 * time.Minute}, nil
 	}
 
 	log.Info("Reconciling a Policy with one or more Gatekeeper constraints")

controllers/secretsync/secret_sync.go

@@ -5,6 +5,7 @@ package secretsync
 
 import (
 	"context"
+	"time"
 
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/equality"
@@ -61,7 +62,7 @@ func (r *SecretReconciler) Reconcile(ctx context.Context, request reconcile.Requ
 	if uninstall.DeploymentIsUninstalling {
 		log.Info("Skipping reconcile because the deployment is in uninstallation mode")
 
-		return reconcile.Result{}, nil
+		return reconcile.Result{RequeueAfter: 5 * time.Minute}, nil
 	}
 
 	reqLogger.Info("Reconciling Secret")

controllers/specsync/policy_spec_sync.go

@@ -6,6 +6,7 @@ package specsync
 import (
 	"context"
 	"fmt"
+	"time"
 
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -70,7 +71,7 @@ func (r *PolicyReconciler) Reconcile(ctx context.Context, request reconcile.Requ
 	if uninstall.DeploymentIsUninstalling {
 		log.Info("Skipping reconcile because the deployment is in uninstallation mode")
 
-		return reconcile.Result{}, nil
+		return reconcile.Result{RequeueAfter: 5 * time.Minute}, nil
 	}
 
 	reqLogger.Info("Reconciling Policy...")

controllers/statussync/policy_status_sync.go

@@ -11,6 +11,7 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"time"
 
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/equality"
@@ -84,7 +85,7 @@ func (r *PolicyReconciler) Reconcile(ctx context.Context, request reconcile.Requ
 	if uninstall.DeploymentIsUninstalling {
 		log.Info("Skipping reconcile because the deployment is in uninstallation mode")
 
-		return reconcile.Result{}, nil
+		return reconcile.Result{RequeueAfter: 5 * time.Minute}, nil
 	}
 
 	reqLogger.Info("Reconciling the policy")

controllers/templatesync/template_sync.go

@@ -10,6 +10,7 @@ import (
 	"fmt"
 	"regexp"
 	"strings"
+	"time"
 
 	"github.com/go-logr/logr"
 	gktemplatesv1 "github.com/open-policy-agent/frameworks/constraint/pkg/apis/templates/v1"
@@ -218,7 +219,7 @@ func (r *PolicyReconciler) Reconcile(ctx context.Context, request reconcile.Requ
 	if uninstall.DeploymentIsUninstalling {
 		log.Info("Skipping reconcile because the deployment is in uninstallation mode")
 
-		return reconcile.Result{}, nil
+		return reconcile.Result{RequeueAfter: 5 * time.Minute}, nil
 	}
 
 	// Handle dependencies that apply to the parent policy

controllers/uninstall/watcher.go

@@ -32,13 +32,6 @@ type reconciler struct {
 func (r *reconciler) Reconcile(ctx context.Context, obj client.ObjectIdentifier) (reconcile.Result, error) {
 	log := log.WithValues("name", obj.Name, "namespace", obj.Namespace)
 
-	if DeploymentIsUninstalling {
-		log.Info("Skipping reconcile because the uninstall flag is already true. " +
-			"To reset the flag, stop this container.")
-
-		return reconcile.Result{}, nil
-	}
-
 	log.Info("Checking the deployment for the annotation " + AnnotationKey)
 
 	deployment, err := r.AppsV1().Deployments(obj.Namespace).Get(ctx, obj.Name, v1.GetOptions{})
@@ -57,14 +50,16 @@ func (r *reconciler) Reconcile(ctx context.Context, obj client.ObjectIdentifier)
 	}
 
 	if deployment.GetAnnotations()[AnnotationKey] == "true" {
-		log.Info("Annotation " + AnnotationKey + " found and was true. Setting the uninstall flag.")
+		log.Info("Annotation " + AnnotationKey + " found and was true. Setting the uninstall flag to true.")
 
 		DeploymentIsUninstalling = true
 
 		return reconcile.Result{}, nil
 	}
 
-	log.Info("Annotation " + AnnotationKey + " not found, or not true. No changes.")
+	log.Info("Annotation " + AnnotationKey + " not found, or not true. Setting the uninstall flag to false.")
+
+	DeploymentIsUninstalling = false
 
 	return reconcile.Result{}, nil
 }