rabbitmq_federation: Add testcase for #10306

[Why]
An upgrade scenario going from RabbitMQ 3.12.x to the upcoming 3.13.0
was shared in issue #10306 to demonstrate that the change of child ID
format broke rolling upgrades when there are existing federated
exchanges.

[How]
The testcase uses 5 nodes:
* one upstream node
* two "old" downstream nodes
* two "new" downstream nodes

The old downstream nodes are used to prepare a 2-node cluster that is
about to be upgraded. The new downstream nodes are added to the cluster
then the old downstream nodes are stopped to simulate that rolling
upgrade.

The child ID format was restored in the previous commit, thus there is
no conversion to handle and the testcase should just work with a fresh
3.13.0+ cluster or with a mixed-version cluster with 3.12.x. It failed
during the preparation of the previous commit to make sure it was
effective.

Author: dumbbell

deps/rabbitmq_federation/BUILD.bazel

@@ -96,7 +96,7 @@ rabbitmq_integration_suite(
         "test/rabbit_federation_test_util.beam",
     ],
     flaky = True,
-    shard_count = 2,
+    shard_count = 3,
 )
 
 rabbitmq_integration_suite(

deps/rabbitmq_federation/test/exchange_SUITE.erl

@@ -30,13 +30,15 @@
 all() ->
     [
       {group, essential},
-      {group, cluster_size_3}
+      {group, cluster_size_3},
+      {group, rolling_upgrade}
     ].
 
 groups() ->
     [
      {essential, [], essential()},
      {cluster_size_3, [], [max_hops]},
+     {rolling_upgrade, [], [child_id_format]},
      {cycle_protection, [], [
                              %% TBD: port from v3.10.x in an Erlang 25-compatible way
                             ]},
@@ -96,6 +98,12 @@ init_per_group(cluster_size_3 = Group, Config) ->
       {rmq_nodes_count, 3}
     ]),
   init_per_group1(Group, Config1);
+init_per_group(rolling_upgrade = Group, Config) ->
+  Config1 = rabbit_ct_helpers:set_config(Config, [
+      {rmq_nodes_count, 5},
+      {rmq_nodes_clustered, false}
+    ]),
+  init_per_group1(Group, Config1);
 init_per_group(Group, Config) ->
   init_per_group1(Group, Config).
 
@@ -539,6 +547,119 @@ lookup_exchange_status(Config) ->
             [key, uri, status, timestamp, id, supervisor, upstream]),
 
   clean_up_federation_related_bits(Config).
+
+child_id_format(Config) ->
+    [UpstreamNode,
+     OldNodeA,
+     NewNodeB,
+     OldNodeC,
+     NewNodeD] = rabbit_ct_broker_helpers:get_node_configs(
+                    Config, nodename),
+
+    %% Create a cluster with the nodes running the old version of RabbitMQ in
+    %% mixed-version testing.
+    %%
+    %% Note: we build this on the assumption that `rabbit_ct_broker_helpers'
+    %% starts nodes this way:
+    %%   Node 1: the primary copy of RabbitMQ the test is started from
+    %%   Node 2: the secondary umbrella (if any)
+    %%   Node 3: the primary copy
+    %%   Node 4: the secondary umbrella
+    %%   ...
+    %%
+    %% Therefore, `UpstreamNode' will use the primary copy, `OldNodeA' the
+    %% secondary umbrella, `NewNodeB' the primary copy, and so on.
+    Config1 = rabbit_ct_broker_helpers:cluster_nodes(
+                Config, [OldNodeA, OldNodeC]),
+
+    %% Prepare the whole federated exchange on that old cluster.
+    UpstreamName = <<"fed_on_upgrade">>,
+    rabbit_ct_broker_helpers:set_parameter(
+      Config1, OldNodeA, <<"federation-upstream">>, UpstreamName,
+      [
+       {<<"uri">>, rabbit_ct_broker_helpers:node_uri(Config1, UpstreamNode)}
+      ]),
+
+    rabbit_ct_broker_helpers:set_policy(
+      Config1, OldNodeA,
+      <<"fed_on_upgrade_policy">>, <<"^fed_">>, <<"all">>,
+      [
+       {<<"federation-upstream-pattern">>, UpstreamName}
+      ]),
+
+    XName = <<"fed_ex_on_upgrade_cluster">>,
+    X = exchange_declare_method(XName, <<"direct">>),
+    {Conn1, Ch1} = rabbit_ct_client_helpers:open_connection_and_channel(
+                     Config1, OldNodeA),
+    ?assertEqual({'exchange.declare_ok'}, declare_exchange(Ch1, X)),
+    rabbit_ct_client_helpers:close_channel(Ch1),
+    rabbit_ct_client_helpers:close_connection(Conn1),
+
+    %% Verify the format of the child ID. In the main branch, the format was
+    %% temporarily a size-2 tuple with a list as the first element. This was
+    %% not kept later and the original ID format is used in old and new nodes.
+    [{Id, _, _, _}] = rabbit_ct_broker_helpers:rpc(
+                        Config1, OldNodeA,
+                        mirrored_supervisor, which_children,
+                        [rabbit_federation_exchange_link_sup_sup]),
+    case Id of
+        %% This is the format we expect everywhere.
+        #exchange{name = #resource{name = XName}} ->
+            %% Verify that the supervisors exist on all nodes.
+            lists:foreach(
+              fun(Node) ->
+                      ?assertMatch(
+                         [{#exchange{name = #resource{name = XName}},
+                           _, _, _}],
+                         rabbit_ct_broker_helpers:rpc(
+                           Config1, Node,
+                           mirrored_supervisor, which_children,
+                           [rabbit_federation_exchange_link_sup_sup]))
+              end, [OldNodeA, OldNodeC]),
+
+            %% Simulate a rolling upgrade by:
+            %% 1. adding new nodes to the old cluster
+            %% 2. stopping the old nodes
+            %%
+            %% After that, the supervisors run on the new code.
+            Config2 = rabbit_ct_broker_helpers:cluster_nodes(
+                        Config1, [OldNodeA, NewNodeB, NewNodeD]),
+            ok = rabbit_ct_broker_helpers:stop_broker(Config2, OldNodeA),
+            ok = rabbit_ct_broker_helpers:reset_node(Config1, OldNodeA),
+            ok = rabbit_ct_broker_helpers:stop_broker(Config2, OldNodeC),
+            ok = rabbit_ct_broker_helpers:reset_node(Config2, OldNodeC),
+
+            %% Verify that the supervisors still use the same IDs.
+            lists:foreach(
+              fun(Node) ->
+                      ?assertMatch(
+                         [{#exchange{name = #resource{name = XName}},
+                           _, _, _}],
+                         rabbit_ct_broker_helpers:rpc(
+                           Config2, Node,
+                           mirrored_supervisor, which_children,
+                           [rabbit_federation_exchange_link_sup_sup]))
+              end, [NewNodeB, NewNodeD]),
+
+            %% Delete the exchange: it should work because the ID format is the
+            %% one expected.
+            %%
+            %% During the transient period where the ID format was changed,
+            %% this would crash with a badmatch because the running
+            %% supervisor's ID would not match the content of the database.
+            {Conn2, Ch2} = rabbit_ct_client_helpers:open_connection_and_channel(
+                             Config2, NewNodeB),
+            ?assertEqual({'exchange.delete_ok'}, delete_exchange(Ch2, XName)),
+            rabbit_ct_client_helpers:close_channel(Ch2),
+            rabbit_ct_client_helpers:close_connection(Conn2);
+
+        %% This is the transient format we are not interested in as it only
+        %% lived in a development branch.
+        {List, #exchange{name = #resource{name = XName}}}
+          when is_list(List) ->
+            {skip, "Testcase skipped with the transiently changed ID format"}
+    end.
+
 %%
 %% Test helpers
 %%