Mnesia: Ask to leave a cluster and retry to join if cluster already consider node a member.

Khepri: no-op. Khepri is less strict already, and rabbit_khepri:can_join would accept a join request from a node that is already a member

(cherry picked from commit dd49cbe)
(cherry picked from commit 6d464f9)

Author: SimonUnge

deps/rabbit/src/rabbit_db_cluster.erl

@@ -214,6 +214,28 @@ join(RemoteNode, NodeType)
             end;
         {ok, already_member} ->
             {ok, already_member};
+        {error, {inconsistent_cluster, Msg}} = Error ->
+            case rabbit_khepri:is_enabled() of
+                true  ->
+                    Error;
+                false ->
+                    %% rabbit_mnesia:can_join_cluster/1 notice inconsistent_cluster,
+                    %% as RemoteNode thinks this node is already in the cluster.
+                    %% Attempt to leave the RemoteNode cluster, the discovery cluster,
+                    %% and simply retry the operation.
+                    rabbit_log:info("Mnesia: node ~tp thinks it's clustered "
+                                    "with node ~tp, but ~tp disagrees. ~tp will ask "
+                                    "to leave the cluster and try again.",
+                                    [RemoteNode, node(), node(), node()]),
+                    try
+                        ok = rabbit_mnesia:leave_discover_cluster(RemoteNode),
+                        join(RemoteNode, NodeType)
+                    catch
+                        _ ->
+                            rabbit_log:error(Msg),
+                            Error
+                    end
+            end;
         {error, _} = Error ->
             Error
     end.

deps/rabbit/src/rabbit_mnesia.erl

@@ -73,7 +73,7 @@
 -export([node_info/0, remove_node_if_mnesia_running/1]).
 
 %% Used internally in `rabbit_db_cluster'.
--export([members/0]).
+-export([members/0, leave_discover_cluster/1]).
 
 %% Used internally in `rabbit_khepri'.
 -export([mnesia_and_msg_store_files/0]).
@@ -179,7 +179,6 @@ can_join_cluster(DiscoveryNode) ->
                     {ok, already_member};
                 false ->
                     Msg = format_inconsistent_cluster_message(DiscoveryNode, node()),
-                    rabbit_log:error(Msg),
                     {error, {inconsistent_cluster, Msg}}
             end
     end.
@@ -894,15 +893,19 @@ remove_node_if_mnesia_running(Node) ->
             end
     end.
 
-leave_cluster() ->
-    case rabbit_nodes:nodes_excl_me(cluster_nodes(all)) of
-        []       -> ok;
-        AllNodes -> case lists:any(fun leave_cluster/1, AllNodes) of
-                        true  -> ok;
-                        false -> e(no_running_cluster_nodes)
-                    end
-    end.
+leave_discover_cluster(DiscoveryNode) ->
+    {ClusterNodes, _, _} = discover_cluster([DiscoveryNode]),
+    leave_cluster(rabbit_nodes:nodes_excl_me(ClusterNodes)).
 
+leave_cluster() ->
+    leave_cluster(rabbit_nodes:nodes_excl_me(cluster_nodes(all))).
+leave_cluster([]) ->
+    ok;
+leave_cluster(Nodes) when is_list(Nodes)  ->
+    case lists:any(fun leave_cluster/1, Nodes) of
+        true  -> ok;
+        false -> e(no_running_cluster_nodes)
+    end;
 leave_cluster(Node) ->
     case rpc:call(Node,
                   rabbit_mnesia, remove_node_if_mnesia_running, [node()]) of