Merge remote-tracking branch 'couchbase/trinity' into phoenix

* couchbase/trinity:
  MB-67857[BP]: Enhance fake_chronicle_kv with seqno support for
  MB-67857[BP]: Reformat fake_chronicle_kv.erl
  MB-67857[BP]: Notify ns_config_events in fake_ns_config
  MB-67857[BP]: Format fake_ns_config
  MB-67857: Only catch menelaus_cbauth_worker noproc errors
  MB-67857: Don't ignore errors from ensure_config_seen_by_nodes
  MB-67908: Avoid leaking cbauth Info in crashes

Change-Id: I11438b1dd1bcaa4d5d0887f66ae4d889c61db97b

Author: Peter-Searby

apps/ns_server/src/cb_creds_rotation.erl

@@ -276,8 +276,8 @@ rotate_password(Type, ProtectionSleep) ->
         fun () ->
                 ns_config:sync_announcements(),
                 chronicle_compat_events:sync(),
-                ns_config_rep:ensure_config_seen_by_nodes(
-                  ns_node_disco:nodes_actual_other(), infinity),
+                ok = ns_config_rep:ensure_config_seen_by_nodes(
+                       ns_node_disco:nodes_actual_other(), infinity),
                 misc:parallel_map(menelaus_cbauth:sync(_),
                                   ns_node_disco:nodes_actual(), infinity)
         end,

apps/ns_server/src/menelaus_cbauth.erl

@@ -78,8 +78,19 @@ sync() ->
 sync(Node) ->
     InternalConnections =
         gen_server:call({?MODULE, Node}, get_internal_connections, infinity),
-    misc:parallel_map(?cut(catch(menelaus_cbauth_worker:sync(_))),
-                      InternalConnections, infinity).
+    %% If the above call succeeds, but a worker exits before we make the below
+    %% sync call, then we would incorrectly crash the caller of this function.
+    %% However, if we simply ignore all exceptions, then other cases like
+    %% timeouts would incorrectly be ignored. As such, we need to catch any
+    %% noproc errors, but allow all other exeptions to be propogated to the
+    %% caller
+    misc:parallel_map(
+      fun (Pid) ->
+              try menelaus_cbauth_worker:sync(Pid)
+              catch exit:{noproc, _} ->
+                      ?log_error("Process ~p no longer exists", [Pid])
+              end
+      end, InternalConnections, infinity).
 
 stats() ->
     InternalConnections = gen_server:call(?MODULE, get_internal_connections),

apps/ns_server/src/menelaus_cbauth_worker.erl

@@ -45,7 +45,7 @@ start_monitor(Label, Version, Pid, Params) ->
                              [Label, Version, Pid, Params], []).
 
 notify(Pid, Info) ->
-    Pid ! {notify, Info}.
+    Pid ! {notify, ?HIDE(Info)}.
 
 collect_stats(Pid) ->
     gen_server:call(Pid, collect_stats).
@@ -82,8 +82,10 @@ handle_info(heartbeat, State = #state{label = Label, connection = Pid,
         ok ->
             {noreply, State}
     end;
-handle_info({notify, Info}, State = #state{label = Label, connection = Pid,
-                                           version = Version}) ->
+handle_info({notify, InfoHidden}, State = #state{label = Label,
+                                                 connection = Pid,
+                                                 version = Version}) ->
+    Info = ?UNHIDE(InfoHidden),
     Method = case Version of
                  internal ->
                      "AuthCacheSvc.UpdateDB";