quorum_queue_member_reconciliation_SUITE: Disconnect node after removing it from cluster

[Why]
We want to disconnect a node that was just reset from the remote cluster
nodes because it may impact peer discovery when the local node is
restarted (as a standalone node).

In particular, it may trigger the recovery code of peer discovery that
tries to repair a cluster after a node lost its disk entirely.

It could also affect the remote cluster as well.

Author: dumbbell

deps/rabbit/test/quorum_queue_member_reconciliation_SUITE.erl

@@ -91,8 +91,21 @@ reset_nodes([], _Leader) ->
     ok;
 reset_nodes([Node| Nodes], Leader) ->
     ok = rabbit_control_helper:command(stop_app, Node),
-    ok = rabbit_control_helper:command(forget_cluster_node, Leader, [atom_to_list(Node)]),
+    case rabbit_control_helper:command(forget_cluster_node, Leader, [atom_to_list(Node)]) of
+        ok -> ok;
+        {error, 69,
+         <<"Error:\n{:not_a_cluster_node, ~c\"The node selected is not in the cluster.\"}">>} -> ok
+    end,
     ok = rabbit_control_helper:command(reset, Node),
+    %% Disconnect explicitly nodes from the node we just reset because it way
+    %% interfere with the restart of RabbitMQ below and peer discovery.
+    ClusterNodes = rpc:call(Leader, rabbit_db_cluster, members, []),
+    lists:foreach(
+      fun(N) ->
+              _ = catch erpc:call(
+                          N,
+                          erlang, disconnect_node, [Node])
+      end, ClusterNodes),
     ok = rabbit_control_helper:command(start_app, Node),
     reset_nodes(Nodes, Leader).