Merge pull request #523 from rabbitmq/pause-reconcile

Pause reconciliation label

Author: ChunyiLyu

bin/kubectl-rabbitmq

@@ -52,6 +52,15 @@ USAGE:
   Enable all feature flags on an INSTANCE
     kubectl rabbitmq enable-all-feature-flags INSTANCE
 
+  Pause reconciliation for an instance
+    kubectl rabbitmq pause-reconciliation INSTANCE
+
+  Resume reconciliation for an instance
+    kubectl rabbitmq resume-reconciliation INSTANCE
+
+  List all instances that has the pause reconciliation label
+    kubectl rabbitmq list-pause-reconciliation-instances
+
   Run perf-test against an instance - you can pass as many perf test parameters as you want
     kubectl rabbitmq perf-test INSTANCE --rate 100
   If you want to monitor perf-test, create the following ServiceMonitor:
@@ -233,6 +242,18 @@ enable_all_feature_flags() {
     kubectl exec "${1}-server-0" -- bash -c "rabbitmqctl list_feature_flags | grep disabled | cut -f 1 | xargs -r -L1 rabbitmqctl enable_feature_flag"
 }
 
+pause-reconciliation() {
+    kubectl label rabbitmqclusters "${1}" rabbitmq.com/pauseReconciliation=true
+}
+
+resume-reconciliation() {
+    kubectl label rabbitmqclusters "${1}" rabbitmq.com/pauseReconciliation-
+}
+
+list-pause-reconciliation-instances() {
+    kubectl get rabbitmqclusters -l rabbitmq.com/pauseReconciliation=true --show-labels
+}
+
 secrets() {
     get_instance_details "$@"
     echo "username: ${username}"
@@ -330,6 +351,25 @@ main() {
         fi
         enable_all_feature_flags "$1"
         ;;
+    "pause-reconciliation")
+        shift 1
+        if [[ "$#" -ne 1 ]]; then
+            usage
+            exit 1
+        fi
+        pause-reconciliation "$1"
+        ;;
+    "resume-reconciliation")
+        shift 1
+        if [[ "$#" -ne 1 ]]; then
+            usage
+            exit 1
+        fi
+        resume-reconciliation "$1"
+        ;;
+    "list-pause-reconciliation-instances")
+        list-pause-reconciliation-instances
+        ;;
     "install-cluster-operator")
         shift 1
         if [[ "$#" -gt 0 ]]; then

bin/kubectl-rabbitmq.bats

@@ -98,6 +98,19 @@ eventually() {
   [[ "$output" == *"secret/bats-default-erlang-cookie"* ]]
 }
 
+@test "pause-and-resume-reconciliation" {
+  kubectl rabbitmq pause-reconciliation bats-default
+  kubectl get rabbitmqclusters.rabbitmq.com bats-default --show-labels | grep rabbitmq.com/pauseReconciliation
+
+  run kubectl rabbitmq list-pause-reconciliation-instances
+
+  [ "$status" -eq 0 ]
+  [[ "$output" == *"bats-default"* ]]
+
+  kubectl rabbitmq resume-reconciliation bats-default
+  kubectl get rabbitmqclusters.rabbitmq.com bats-default --show-labels | grep none
+}
+
 @test "secrets prints secrets of default-user" {
   run kubectl rabbitmq secrets bats-default
 

controllers/rabbitmqcluster_controller.go

@@ -48,8 +48,9 @@ var (
 )
 
 const (
-	ownerKey  = ".metadata.controller"
-	ownerKind = "RabbitmqCluster"
+	ownerKey                 = ".metadata.controller"
+	ownerKind                = "RabbitmqCluster"
+	pauseReconciliationLabel = "rabbitmq.com/pauseReconciliation"
 )
 
 // RabbitmqClusterReconciler reconciles a RabbitmqCluster object
@@ -101,6 +102,23 @@ func (r *RabbitmqClusterReconciler) Reconcile(req ctrl.Request) (ctrl.Result, er
 		return ctrl.Result{}, r.prepareForDeletion(ctx, rabbitmqCluster)
 	}
 
+	// exit if pause reconciliation label is set to true
+	if v, ok := rabbitmqCluster.Labels[pauseReconciliationLabel]; ok && v == "true" {
+		logger.Info("Not reconciling RabbitmqCluster",
+			"namespace", rabbitmqCluster.Namespace,
+			"name", rabbitmqCluster.Name)
+		r.Recorder.Event(rabbitmqCluster, corev1.EventTypeWarning,
+			"PausedReconciliation", fmt.Sprintf("label '%s' is set to true", pauseReconciliationLabel))
+
+		rabbitmqCluster.Status.SetCondition(status.NoWarnings, corev1.ConditionFalse, "reconciliation paused")
+		if writerErr := r.Status().Update(ctx, rabbitmqCluster); writerErr != nil {
+			r.Log.Error(writerErr, "Error trying to Update NoWarnings condition state",
+				"namespace", rabbitmqCluster.Namespace,
+				"name", rabbitmqCluster.Name)
+		}
+		return ctrl.Result{}, nil
+	}
+
 	// Ensure the resource have a deletion marker
 	if err := r.addFinalizerIfNeeded(ctx, rabbitmqCluster); err != nil {
 		return ctrl.Result{}, err