rabbit_peer_discovery: Retry rpc calls

Author: dumbbell

deps/rabbit/src/rabbit_peer_discovery.erl

@@ -380,6 +380,8 @@ check_discovered_nodes_list_validity(DiscoveredNodes, BadNodeType)
 %%
 %% @private
 
+-define(PEER_PARENT_NODE_KEY(Peer), {?MODULE, peer_parent_node, Peer}).
+
 query_node_props(Nodes) when Nodes =/= [] ->
     ThisNode = node(),
     {Prefix, Suffix} = rabbit_nodes_common:parts(ThisNode),
@@ -444,13 +446,21 @@ query_node_props(Nodes) when Nodes =/= [] ->
                [Peer],
                #{domain => ?RMQLOG_DOMAIN_PEER_DISC}),
             try
+                _ = peer:call(
+                      Pid,
+                      persistent_term, put,
+                      [?PEER_PARENT_NODE_KEY(Peer), ThisNode]),
+
                 NodesAndProps1 = (
                   do_query_node_props([ThisNode]) ++
                   peer:call(
                     Pid, ?MODULE, do_query_node_props, [Nodes], 180000)),
                 NodesAndProps2 = sort_nodes_and_props(NodesAndProps1),
                 NodesAndProps2
             after
+                _ = peer:call(
+                      Pid,
+                      persistent_term, erase, [?PEER_PARENT_NODE_KEY(Peer)]),
                 peer:stop(Pid)
             end;
         {error, _} = Error ->
@@ -579,7 +589,12 @@ do_query_node_props(Nodes) when Nodes =/= [] ->
 
     %% TODO: Replace with `rabbit_nodes:list_members/0' when the oldest
     %% supported version has it.
-    MembersPerNode = erpc:multicall(Nodes, rabbit_nodes, all, []),
+    MembersPerNode = [try
+                          {ok, erpc_call(Node, rabbit_nodes, all, [])}
+                      catch
+                          Class:Reason ->
+                              {Class, Reason}
+                      end || Node <- Nodes],
     query_node_props1(Nodes, MembersPerNode, []).
 
 query_node_props1(
@@ -588,7 +603,7 @@ query_node_props1(
     NodesAndProps1 = [NodeAndProps | NodesAndProps],
     query_node_props1(Nodes, MembersPerNode, NodesAndProps1);
 query_node_props1(
-  [Node | Nodes], [{error, _} = Error | MembersPerNode], NodesAndProps) ->
+  [Node | Nodes], [{_, _} = Error | MembersPerNode], NodesAndProps) ->
     %% We consider that an error means the remote node is unreachable or not
     %% ready. Therefore, we exclude it from the list of discovered nodes as we
     %% won't be able to join it anyway.
@@ -604,7 +619,7 @@ query_node_props1([], [], NodesAndProps) ->
 
 query_node_props2([{Node, Members} | Rest], NodesAndProps) ->
     try
-        erpc:call(
+        erpc_call(
           Node, logger, debug,
           ["Peer discovery: temporary hidden node '~ts' queries properties "
            "from node '~ts'", [node(), Node]]),
@@ -656,10 +671,10 @@ query_node_props2([], NodesAndProps) ->
 %% @private
 
 get_node_start_time(Node, Unit) ->
-    NativeStartTime = erpc:call(Node, erlang, system_info, [start_time]),
-    TimeOffset = erpc:call(Node, erlang, time_offset, []),
+    NativeStartTime = erpc_call(Node, erlang, system_info, [start_time]),
+    TimeOffset = erpc_call(Node, erlang, time_offset, []),
     SystemStartTime = NativeStartTime + TimeOffset,
-    StartTime = erpc:call(
+    StartTime = erpc_call(
                   Node, erlang, convert_time_unit,
                   [SystemStartTime, native, Unit]),
     StartTime.
@@ -679,12 +694,30 @@ is_node_db_ready(Node) when Node =:= node() ->
     true;
 is_node_db_ready(Node) ->
     try
-        erpc:call(Node, rabbit_db, is_init_finished, [])
+        erpc_call(Node, rabbit_db, is_init_finished, [])
     catch
         _:{exception, undef, [{rabbit_db, is_init_finished, _, _} | _]} ->
             undefined
     end.
 
+erpc_call(Node, Mod, Fun, Args) ->
+    try
+        erpc:call(Node, Mod, Fun, Args)
+    catch
+        error:{erpc, Reason}:Stacktrace ->
+            Peer = node(),
+            ParentNode = persistent_term:get(
+                           ?PEER_PARENT_NODE_KEY(Peer), Peer),
+            _ = catch erpc:call(
+                        ParentNode,
+                        logger, alert, % XXX
+                        ["Peer discovery: temporary hidden node '~ts' failed to connect to '~ts': ~0p",
+                         [Peer, Node, Reason]]),
+            erlang:raise(error, {erpc, Reason}, Stacktrace)
+            %timer:sleep(200),
+            %erpc_call(Node, Mod, Fun, Args)
+    end.
+
 -spec sort_nodes_and_props(NodesAndProps) ->
     SortedNodesAndProps when
       NodesAndProps :: [node_and_props()],