Merge pull request #12853 from rabbitmq/mirrored-sup-retry-update-after-db-timeout

mirrored_supervisor: Catch timeout from Khepri in `hanlde_info/2`

Author: dumbbell

deps/rabbit/src/mirrored_supervisor.erl

@@ -345,7 +345,7 @@ handle_info({'DOWN', _Ref, process, Pid, _Reason},
                            child_order = ChildOrder}) ->
     %% No guarantee pg will have received the DOWN before us.
     R = case lists:sort(pg:get_members(Group)) -- [Pid] of
-            [O | _] -> ChildSpecs = update_all(O, Pid),
+            [O | _] -> ChildSpecs = retry_update_all(O, Pid),
                        [start(Delegate, ChildSpec)
                         || ChildSpec <- restore_child_order(ChildSpecs,
                                                             ChildOrder)];
@@ -428,6 +428,22 @@ check_stop(Group, Delegate, Id) ->
 
 id({Id, _, _, _, _, _}) -> Id.
 
+retry_update_all(O, Pid) ->
+    retry_update_all(O, Pid, 10000).
+
+retry_update_all(O, Pid, TimeLeft) when TimeLeft > 0 ->
+    case update_all(O, Pid) of
+        List when is_list(List) ->
+            List;
+        {error, timeout} ->
+            Sleep = 200,
+            TimeLeft1 = TimeLeft - Sleep,
+            timer:sleep(Sleep),
+            retry_update_all(O, Pid, TimeLeft1)
+    end;
+retry_update_all(O, Pid, _TimeLeft) ->
+    update_all(O, Pid).
+
 update_all(Overall, OldOverall) ->
     rabbit_db_msup:update_all(Overall, OldOverall).
 

deps/rabbit/src/rabbit_db_msup.erl

@@ -226,9 +226,10 @@ find_mirror_in_khepri(Group, Id) ->
 %% update_all().
 %% -------------------------------------------------------------------
 
--spec update_all(Overall, Overall) -> [ChildSpec] when
+-spec update_all(Overall, Overall) -> [ChildSpec] | Error when
       Overall :: pid(),
-      ChildSpec :: supervisor2:child_spec().
+      ChildSpec :: supervisor2:child_spec(),
+      Error :: {error, any()}.
 
 update_all(Overall, OldOverall) ->
     rabbit_khepri:handle_fallback(