rabbitmq_ct_broker_helpers: Use node 2 as the cluster seed node

[Why]
When running mixed-version tests, nodes 1/3/5/... are using the primary
umbrella, so usually the newest version. Nodes 2/4/6/... are using the
secondary umbrella, thus the old version.

When clustering, we used to use node 1 (running a new version) as the
seed node, meaning other nodes would join it.

This complicates things with feature flags because we have to make sure
that we start node 1 with new stable feature flags disabled to allow old
nodes to join.

This is also a problem with Khepri machine versions because the cluster
would start with the latest version, which old nodes might not have.

[How]
This patch changes the logic to use a node running the secondary
umbrella as the seed node instead. If there is no node running it, we
pick the first node as before.

Author: dumbbell

deps/rabbitmq_ct_helpers/src/rabbit_ct_broker_helpers.erl

@@ -826,7 +826,9 @@ do_start_rabbitmq_node(Config, NodeConfig, I) ->
                 {ok, _} ->
                     NodeConfig1 = rabbit_ct_helpers:set_config(
                                     NodeConfig,
-                                    [{effective_srcdir, SrcDir},
+                                    [{use_secondary_umbrella,
+                                      UseSecondaryUmbrella},
+                                     {effective_srcdir, SrcDir},
                                      {make_vars_for_node_startup, MakeVars}]),
                     query_node(Config, NodeConfig1);
                 _ ->
@@ -936,13 +938,33 @@ maybe_cluster_nodes(Config) ->
     end.
 
 cluster_nodes(Config) ->
-    [NodeConfig1 | NodeConfigs] = get_node_configs(Config),
-    cluster_nodes1(Config, NodeConfig1, NodeConfigs).
+    Nodenames = get_node_configs(Config, nodename),
+    cluster_nodes(Config, Nodenames).
 
 cluster_nodes(Config, Nodes) ->
-    [NodeConfig1 | NodeConfigs] = [
-      get_node_config(Config, Node) || Node <- Nodes],
-    cluster_nodes1(Config, NodeConfig1, NodeConfigs).
+    NodeConfigs = [get_node_config(Config, Node) || Node <- Nodes],
+    Search = lists:search(
+               fun(NodeConfig) ->
+                       rabbit_ct_helpers:get_config(
+                         NodeConfig, use_secondary_umbrella, false)
+               end, NodeConfigs),
+    case Search of
+        {value, SecNodeConfig} ->
+            NodeConfigs1 = NodeConfigs -- [SecNodeConfig],
+            Nodename = ?config(nodename, SecNodeConfig),
+            ct:pal(
+              "Using secondary-umbrella-based node ~s as the cluster seed "
+              "node",
+              [Nodename]),
+            cluster_nodes1(Config, SecNodeConfig, NodeConfigs1);
+        false ->
+            [NodeConfig | NodeConfigs1] = NodeConfigs,
+            Nodename = ?config(nodename, NodeConfig),
+            ct:pal(
+              "Using node ~s as the cluster seed node",
+              [Nodename]),
+            cluster_nodes1(Config, NodeConfig, NodeConfigs1)
+    end.
 
 cluster_nodes1(Config, NodeConfig1, [NodeConfig2 | Rest]) ->
     case cluster_nodes(Config, NodeConfig2, NodeConfig1) of