rabbitmq_ct_broker_helpers: Use node 2 as the cluster seed node

[Why]
When running mixed-version tests, nodes 1/3/5/... are using the primary
umbrella, so usually the newest version. Nodes 2/4/6/... are using the
secondary umbrella, thus the old version.

When clustering, we used to use node 1 (running a new version) as the
seed node, meaning other nodes would join it.

This complicates things with feature flags because we have to make sure
that we start node 1 with new stable feature flags disabled to allow old
nodes to join.

This is also a problem with Khepri machine versions because the cluster
would start with the latest version, which old nodes might not have.

[How]
This patch changes the logic to use a node running the secondary
umbrella as the seed node instead. If there is no node running it, we
pick the first node as before.

Author: dumbbell

deps/rabbit/test/feature_flags_SUITE.erl

@@ -891,7 +891,7 @@ clustering_ok_with_ff_enabled_on_some_nodes(Config) ->
             ok
     end,
 
-    ?assertEqual(Config, rabbit_ct_broker_helpers:cluster_nodes(Config)),
+    ?assertEqual(Config, rabbit_ct_broker_helpers:cluster_nodes(Config, 0)),
 
     log_feature_flags_of_all_nodes(Config),
     case FFSubsysOk of
@@ -987,7 +987,7 @@ clustering_denied_with_new_ff_enabled(Config) ->
         false -> ok
     end,
 
-    ?assertMatch({skip, _}, rabbit_ct_broker_helpers:cluster_nodes(Config)),
+    ?assertMatch({skip, _}, rabbit_ct_broker_helpers:cluster_nodes(Config, 0)),
 
     log_feature_flags_of_all_nodes(Config),
     case FFSubsysOk of
@@ -1049,7 +1049,7 @@ clustering_ok_with_new_ff_enabled_from_plugin_on_some_nodes(Config) ->
         false -> ok
     end,
 
-    ?assertEqual(Config, rabbit_ct_broker_helpers:cluster_nodes(Config)),
+    ?assertEqual(Config, rabbit_ct_broker_helpers:cluster_nodes(Config, 0)),
 
     log_feature_flags_of_all_nodes(Config),
     case FFSubsysOk of

deps/rabbitmq_ct_helpers/src/rabbit_ct_broker_helpers.erl

@@ -23,7 +23,7 @@
     stop_rabbitmq_nodes/1,
     stop_rabbitmq_nodes_on_vms/1,
     rewrite_node_config_file/2,
-    cluster_nodes/1, cluster_nodes/2,
+    cluster_nodes/1, cluster_nodes/2, cluster_nodes/3,
 
     setup_meck/1,
     setup_meck/2,
@@ -826,7 +826,9 @@ do_start_rabbitmq_node(Config, NodeConfig, I) ->
                 {ok, _} ->
                     NodeConfig1 = rabbit_ct_helpers:set_config(
                                     NodeConfig,
-                                    [{effective_srcdir, SrcDir},
+                                    [{use_secondary_umbrella,
+                                      UseSecondaryUmbrella},
+                                     {effective_srcdir, SrcDir},
                                      {make_vars_for_node_startup, MakeVars}]),
                     query_node(Config, NodeConfig1);
                 _ ->
@@ -936,23 +938,52 @@ maybe_cluster_nodes(Config) ->
     end.
 
 cluster_nodes(Config) ->
-    [NodeConfig1 | NodeConfigs] = get_node_configs(Config),
-    cluster_nodes1(Config, NodeConfig1, NodeConfigs).
+    Nodenames = get_node_configs(Config, nodename),
+    cluster_nodes(Config, Nodenames).
+
+cluster_nodes(Config, Nodes) when is_list(Nodes) ->
+    NodeConfigs = [get_node_config(Config, Node) || Node <- Nodes],
+    Search = lists:search(
+               fun(NodeConfig) ->
+                       rabbit_ct_helpers:get_config(
+                         NodeConfig, use_secondary_umbrella, false)
+               end, NodeConfigs),
+    case Search of
+        {value, SecNodeConfig} ->
+            NodeConfigs1 = NodeConfigs -- [SecNodeConfig],
+            Nodename = ?config(nodename, SecNodeConfig),
+            ct:pal(
+              "Using secondary-umbrella-based node ~s as the cluster seed "
+              "node",
+              [Nodename]),
+            cluster_nodes1(Config, SecNodeConfig, NodeConfigs1);
+        false ->
+            [NodeConfig | NodeConfigs1] = NodeConfigs,
+            Nodename = ?config(nodename, NodeConfig),
+            ct:pal(
+              "Using node ~s as the cluster seed node",
+              [Nodename]),
+            cluster_nodes1(Config, NodeConfig, NodeConfigs1)
+    end;
+cluster_nodes(Config, SeedNode) ->
+    Nodenames = get_node_configs(Config, nodename),
+    cluster_nodes(Config, SeedNode, Nodenames).
 
-cluster_nodes(Config, Nodes) ->
-    [NodeConfig1 | NodeConfigs] = [
-      get_node_config(Config, Node) || Node <- Nodes],
-    cluster_nodes1(Config, NodeConfig1, NodeConfigs).
+cluster_nodes(Config, SeedNode, Nodes) ->
+    SeedNodeConfig = get_node_config(Config, SeedNode),
+    NodeConfigs = [get_node_config(Config, Node) || Node <- Nodes],
+    NodeConfigs1 = NodeConfigs -- [SeedNodeConfig],
+    cluster_nodes1(Config, SeedNodeConfig, NodeConfigs1).
 
 cluster_nodes1(Config, NodeConfig1, [NodeConfig2 | Rest]) ->
-    case cluster_nodes(Config, NodeConfig2, NodeConfig1) of
+    case do_cluster_nodes(Config, NodeConfig2, NodeConfig1) of
         ok    -> cluster_nodes1(Config, NodeConfig1, Rest);
         Error -> Error
     end;
 cluster_nodes1(Config, _, []) ->
     Config.
 
-cluster_nodes(Config, NodeConfig1, NodeConfig2) ->
+do_cluster_nodes(Config, NodeConfig1, NodeConfig2) ->
     Nodename1 = ?config(nodename, NodeConfig1),
     Nodename2 = ?config(nodename, NodeConfig2),
     Cmds = [