Skip to content

Commit b351149

Browse files
authored
Merge pull request #12558 from rabbitmq/loic-make-ci-mixed-versions
Make CI: Mixed version testing
2 parents 8b79ac7 + e89e1fa commit b351149

File tree

10 files changed

+160
-19
lines changed

10 files changed

+160
-19
lines changed

.github/workflows/test-make-target.yaml

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,10 @@ on:
1111
metadata_store:
1212
required: true
1313
type: string
14+
mixed_clusters:
15+
required: false
16+
default: false
17+
type: boolean
1418
make_target:
1519
required: true
1620
type: string
@@ -41,6 +45,31 @@ jobs:
4145
# restricted to the build jobs to avoid duplication in output.
4246
disable_problem_matchers: true
4347

48+
- name: MIXED CLUSTERS - FETCH SIGNING KEYS
49+
uses: dsaltares/fetch-gh-release-asset@master
50+
if: inputs.mixed_clusters
51+
with:
52+
repo: rabbitmq/signing-keys
53+
file: rabbitmq-release-signing-key.asc
54+
55+
- name: MIXED CLUSTERS - FETCH PREVIOUS VERSION
56+
id: fetch_secondary_dist
57+
uses: dsaltares/fetch-gh-release-asset@master
58+
if: inputs.mixed_clusters
59+
with:
60+
regex: true
61+
file: "rabbitmq-server-generic-unix-[\\d.]*\\.tar.xz"
62+
target: ./
63+
64+
- name: MIXED CLUSTERS - SETUP SECONDARY_DIST
65+
if: inputs.mixed_clusters
66+
run: |
67+
gpg --import rabbitmq-release-signing-key.asc
68+
gpg --verify rabbitmq-server-generic-unix-*.asc rabbitmq-server-generic-unix-*.tar.xz
69+
tar xf rabbitmq-server-generic-unix-*.tar.xz
70+
71+
echo "SECONDARY_DIST=${GITHUB_WORKSPACE}/rabbitmq_server-`echo -n ${{ steps.fetch_secondary_dist.outputs.version }} | sed s/v//`" >> $GITHUB_ENV
72+
4473
- name: SETUP DOTNET (rabbit)
4574
uses: actions/setup-dotnet@v4
4675
if: inputs.plugin == 'rabbit'
@@ -74,7 +103,7 @@ jobs:
74103
if: always()
75104
uses: actions/upload-artifact@v4
76105
with:
77-
name: CT logs (${{ inputs.plugin }} ${{ inputs.make_target }} OTP-${{ inputs.erlang_version }} ${{ inputs.metadata_store }})
106+
name: CT logs (${{ inputs.plugin }} ${{ inputs.make_target }} OTP-${{ inputs.erlang_version }} ${{ inputs.metadata_store }}${{ inputs.mixed_clusters && ' mixed' || '' }})
78107
path: |
79108
logs/
80109
# !logs/**/log_private

.github/workflows/test-make-tests.yaml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,9 @@ on:
1111
metadata_store:
1212
required: true
1313
type: string
14+
mixed_clusters:
15+
required: true
16+
type: boolean
1417
jobs:
1518
test-rabbit:
1619
name: Test rabbit
@@ -33,6 +36,7 @@ jobs:
3336
erlang_version: ${{ inputs.erlang_version }}
3437
elixir_version: ${{ inputs.elixir_version }}
3538
metadata_store: ${{ inputs.metadata_store }}
39+
mixed_clusters: ${{ inputs.mixed_clusters }}
3640
make_target: ${{ matrix.make_target }}
3741
plugin: rabbit
3842

@@ -43,6 +47,7 @@ jobs:
4347
erlang_version: ${{ inputs.erlang_version }}
4448
elixir_version: ${{ inputs.elixir_version }}
4549
metadata_store: ${{ inputs.metadata_store }}
50+
mixed_clusters: ${{ inputs.mixed_clusters }}
4651
make_target: parallel-ct-set-1
4752
plugin: rabbitmq_mqtt
4853

@@ -55,6 +60,7 @@ jobs:
5560
erlang_version: ${{ inputs.erlang_version }}
5661
elixir_version: ${{ inputs.elixir_version }}
5762
metadata_store: ${{ inputs.metadata_store }}
63+
mixed_clusters: ${{ inputs.mixed_clusters }}
5864
make_target: ct-config_schema ct-unit
5965
plugin: rabbitmq_peer_discovery_aws
6066

@@ -110,5 +116,6 @@ jobs:
110116
erlang_version: ${{ inputs.erlang_version }}
111117
elixir_version: ${{ inputs.elixir_version }}
112118
metadata_store: ${{ inputs.metadata_store }}
119+
mixed_clusters: ${{ inputs.mixed_clusters }}
113120
make_target: tests
114121
plugin: ${{ matrix.plugin }}

.github/workflows/test-make.yaml

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,27 @@ jobs:
7474
erlang_version: ${{ matrix.erlang_version }}
7575
elixir_version: ${{ matrix.elixir_version }}
7676
metadata_store: ${{ matrix.metadata_store }}
77+
mixed_clusters: false
78+
79+
test-mixed-clusters:
80+
name: Test mixed clusters
81+
strategy:
82+
fail-fast: false
83+
matrix:
84+
erlang_version:
85+
- '26'
86+
# - '27'
87+
elixir_version:
88+
- '1.17'
89+
metadata_store:
90+
- mnesia
91+
- khepri
92+
uses: ./.github/workflows/test-make-tests.yaml
93+
with:
94+
erlang_version: ${{ matrix.erlang_version }}
95+
elixir_version: ${{ matrix.elixir_version }}
96+
metadata_store: ${{ matrix.metadata_store }}
97+
mixed_clusters: true
7798

7899
type-check:
79100
name: Type check

.github/workflows/test-mixed-versions.yaml

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ name: Test Mixed Version Clusters
22
on:
33
push:
44
branches:
5-
- main
65
- v4.0.x
76
- v3.13.x
87
- bump-otp-*
@@ -21,7 +20,6 @@ on:
2120
- '*.bzl'
2221
- '*.bazel'
2322
- .github/workflows/test-mixed-versions.yaml
24-
pull_request:
2523
concurrency:
2624
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
2725
cancel-in-progress: true

deps/rabbit/test/cluster_upgrade_SUITE.erl

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ init_per_testcase(Testcase, Config) ->
5555
Config1 = rabbit_ct_helpers:set_config(Config, [
5656
{rmq_nodename_suffix, Testcase},
5757
{rmq_nodes_count, 3},
58-
{force_secondary_umbrella, true}
58+
{force_secondary, true}
5959
]),
6060
Config2 = rabbit_ct_helpers:run_steps(Config1,
6161
rabbit_ct_broker_helpers:setup_steps() ++
@@ -139,7 +139,7 @@ upgrade_cluster(Config) ->
139139
|| N <- Cluster],
140140
ct:pal(?LOW_IMPORTANCE, "Restarting cluster ~p", [Cluster]),
141141
Config1 = rabbit_ct_helpers:set_config(
142-
Config, {force_secondary_umbrella, false}),
142+
Config, {force_secondary, false}),
143143
[ok = rabbit_ct_broker_helpers:async_start_node(Config1, N)
144144
|| N <- Cluster],
145145
[ok = rabbit_ct_broker_helpers:wait_for_async_start_node(N)

deps/rabbit/test/feature_flags_SUITE.erl

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,7 @@ init_per_group(registry, Config) ->
126126
logger:set_primary_config(level, debug),
127127
rabbit_ct_helpers:run_steps(Config, []);
128128
init_per_group(feature_flags_v2, Config) ->
129+
%% @todo Remove this entirely as that FF became required in 3.12.
129130
%% `feature_flags_v2' is now required and won't work in mixed-version
130131
%% clusters if the other version doesn't support it.
131132
case rabbit_ct_helpers:is_mixed_versions() of
@@ -267,6 +268,7 @@ init_per_testcase(Testcase, Config) ->
267268
Config2 = rabbit_ct_helpers:set_config(
268269
Config1,
269270
[{rmq_nodename_suffix, Testcase},
271+
{secondary_enabled_plugins, "my_plugin"},
270272
{tcp_ports_base, {skip_n_nodes,
271273
TestNumber * ClusterSize}},
272274
{net_ticktime, 5}

deps/rabbit_common/mk/rabbitmq-dist.mk

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
.PHONY: dist test-dist do-dist cli-scripts cli-escripts clean-dist
22

3-
DIST_DIR = plugins
4-
CLI_SCRIPTS_DIR = sbin
5-
CLI_ESCRIPTS_DIR = escript
3+
DIST_DIR ?= $(CURDIR)/plugins
4+
CLI_SCRIPTS_DIR ?= $(CURDIR)/sbin
5+
CLI_ESCRIPTS_DIR ?= $(CURDIR)/escript
66
MIX = echo y | mix
77

88
# Set $(DIST_AS_EZS) to a non-empty value to enable the packaging of

deps/rabbit_common/mk/rabbitmq-run.mk

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ TEST_TMPDIR ?= $(TMPDIR)/rabbitmq-test-instances
1919
endif
2020

2121
# Location of the scripts controlling the broker.
22-
RABBITMQ_SCRIPTS_DIR ?= $(CURDIR)/sbin
22+
RABBITMQ_SCRIPTS_DIR ?= $(CLI_SCRIPTS_DIR)
2323

2424
ifeq ($(PLATFORM),msys2)
2525
RABBITMQ_PLUGINS ?= $(RABBITMQ_SCRIPTS_DIR)/rabbitmq-plugins.bat
@@ -39,7 +39,7 @@ export RABBITMQ_SCRIPTS_DIR RABBITMQCTL RABBITMQ_PLUGINS RABBITMQ_SERVER RABBITM
3939
export MAKE
4040

4141
# We need to pass the location of codegen to the Java client ant
42-
# process.
42+
# process. @todo Delete?
4343
CODEGEN_DIR = $(DEPS_DIR)/rabbitmq_codegen
4444
PYTHONPATH = $(CODEGEN_DIR)
4545
export PYTHONPATH
@@ -90,7 +90,7 @@ ifdef PLUGINS_FROM_DEPS_DIR
9090
RMQ_PLUGINS_DIR = $(DEPS_DIR)
9191
DIST_ERL_LIBS = $(ERL_LIBS)
9292
else
93-
RMQ_PLUGINS_DIR = $(CURDIR)/$(DIST_DIR)
93+
RMQ_PLUGINS_DIR = $(DIST_DIR)
9494
# We do not want to add apps/ or deps/ to ERL_LIBS
9595
# when running the release from dist. The `plugins`
9696
# directory is added to ERL_LIBS by rabbitmq-env.

deps/rabbitmq_ct_helpers/src/rabbit_ct_broker_helpers.erl

Lines changed: 75 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -433,6 +433,7 @@ start_rabbitmq_node(Master, Config, NodeConfig, I) ->
433433
%% It's unlikely we'll ever succeed to start RabbitMQ.
434434
Master ! {self(), Error},
435435
unlink(Master);
436+
%% @todo This might not work right now in at least some cases...
436437
{skip, _} ->
437438
%% Try again with another TCP port numbers base.
438439
NodeConfig4 = move_nonworking_nodedir_away(NodeConfig3),
@@ -506,6 +507,7 @@ tcp_port_base_for_broker0(Config, I, PortsCount) ->
506507
tcp_port_base_for_broker1(Base, I, PortsCount) ->
507508
Base + I * PortsCount * ?NODE_START_ATTEMPTS.
508509

510+
%% @todo Refactor to simplify this...
509511
update_tcp_ports_in_rmq_config(NodeConfig, [tcp_port_amqp = Key | Rest]) ->
510512
NodeConfig1 = rabbit_ct_helpers:merge_app_env(NodeConfig,
511513
{rabbit, [{tcp_listeners, [?config(Key, NodeConfig)]}]}),
@@ -626,21 +628,52 @@ write_config_file(Config, NodeConfig, _I) ->
626628
ConfigFile ++ "\": " ++ file:format_error(Reason)}
627629
end.
628630

631+
-define(REQUIRED_FEATURE_FLAGS, [
632+
%% Required in 3.11:
633+
"virtual_host_metadata,"
634+
"quorum_queue,"
635+
"implicit_default_bindings,"
636+
"maintenance_mode_status,"
637+
"user_limits,"
638+
%% Required in 3.12:
639+
"stream_queue,"
640+
"classic_queue_type_delivery_support,"
641+
"tracking_records_in_ets,"
642+
"stream_single_active_consumer,"
643+
"listener_records_in_ets,"
644+
"feature_flags_v2,"
645+
"direct_exchange_routing_v2,"
646+
"classic_mirrored_queue_version," %% @todo Missing in FF docs!!
647+
%% Required in 3.12 in rabbitmq_management_agent:
648+
% "drop_unroutable_metric,"
649+
% "empty_basic_get_metric,"
650+
%% Required in 4.0:
651+
"stream_sac_coordinator_unblock_group,"
652+
"restart_streams,"
653+
"stream_update_config_command,"
654+
"stream_filtering,"
655+
"message_containers" %% @todo Update FF docs!! It *is* required.
656+
]).
657+
629658
do_start_rabbitmq_node(Config, NodeConfig, I) ->
630659
WithPlugins0 = rabbit_ct_helpers:get_config(Config,
631-
broker_with_plugins),
660+
broker_with_plugins), %% @todo This is probably not used.
632661
WithPlugins = case is_list(WithPlugins0) of
633662
true -> lists:nth(I + 1, WithPlugins0);
634663
false -> WithPlugins0
635664
end,
636665
ForceUseSecondary = rabbit_ct_helpers:get_config(
637-
Config, force_secondary_umbrella, undefined),
666+
Config, force_secondary, undefined),
638667
CanUseSecondary = case ForceUseSecondary of
639668
undefined ->
640669
(I + 1) rem 2 =:= 0;
641670
Override when is_boolean(Override) ->
642671
Override
643672
end,
673+
UseSecondaryDist = case ?config(secondary_dist, Config) of
674+
false -> false;
675+
_ -> CanUseSecondary
676+
end,
644677
UseSecondaryUmbrella = case ?config(secondary_umbrella, Config) of
645678
false -> false;
646679
_ -> CanUseSecondary
@@ -686,8 +719,10 @@ do_start_rabbitmq_node(Config, NodeConfig, I) ->
686719
StartWithPluginsDisabled = rabbit_ct_helpers:get_config(
687720
Config, start_rmq_with_plugins_disabled),
688721
ExtraArgs2 = case StartWithPluginsDisabled of
689-
true -> ["LEAVE_PLUGINS_DISABLED=yes" | ExtraArgs1];
690-
_ -> ExtraArgs1
722+
true ->
723+
["LEAVE_PLUGINS_DISABLED=1" | ExtraArgs1];
724+
_ ->
725+
ExtraArgs1
691726
end,
692727
KeepPidFile = rabbit_ct_helpers:get_config(
693728
Config, keep_pid_file_on_exit),
@@ -731,7 +766,30 @@ do_start_rabbitmq_node(Config, NodeConfig, I) ->
731766
{"RABBITMQ_PLUGINS=~ts/rabbitmq-plugins", [SecScriptsDir]}
732767
| ExtraArgs4];
733768
false ->
734-
ExtraArgs4
769+
case UseSecondaryDist of
770+
true ->
771+
SecondaryDist = ?config(secondary_dist, Config),
772+
SecondaryEnabledPlugins = case {
773+
StartWithPluginsDisabled,
774+
?config(secondary_enabled_plugins, Config),
775+
filename:basename(SrcDir)
776+
} of
777+
{true, _, _} -> "";
778+
{_, undefined, "rabbit"} -> "";
779+
{_, undefined, SrcPlugin} -> SrcPlugin;
780+
{_, SecondaryEnabledPlugins0, _} -> SecondaryEnabledPlugins0
781+
end,
782+
[{"DIST_DIR=~ts/plugins", [SecondaryDist]},
783+
{"CLI_SCRIPTS_DIR=~ts/sbin", [SecondaryDist]},
784+
{"CLI_ESCRIPTS_DIR=~ts/escript", [SecondaryDist]},
785+
{"RABBITMQ_SCRIPTS_DIR=~ts/sbin", [SecondaryDist]},
786+
{"RABBITMQ_SERVER=~ts/sbin/rabbitmq-server", [SecondaryDist]},
787+
{"RABBITMQ_ENABLED_PLUGINS=~ts", [SecondaryEnabledPlugins]},
788+
{"RABBITMQ_FEATURE_FLAGS=~ts", [?REQUIRED_FEATURE_FLAGS]}
789+
| ExtraArgs4];
790+
false ->
791+
ExtraArgs4
792+
end
735793
end,
736794
MakeVars = [
737795
{"RABBITMQ_NODENAME=~ts", [Nodename]},
@@ -1285,6 +1343,10 @@ rabbitmqctl(Config, Node, Args, Timeout) ->
12851343
CanUseSecondary = (I + 1) rem 2 =:= 0,
12861344
BazelRunSecCmd = rabbit_ct_helpers:get_config(
12871345
Config, rabbitmq_run_secondary_cmd),
1346+
UseSecondaryDist = case ?config(secondary_dist, Config) of
1347+
false -> false;
1348+
_ -> CanUseSecondary
1349+
end,
12881350
UseSecondaryUmbrella = case ?config(secondary_umbrella, Config) of
12891351
false ->
12901352
case BazelRunSecCmd of
@@ -1327,7 +1389,14 @@ rabbitmqctl(Config, Node, Args, Timeout) ->
13271389
"rabbitmqctl"])
13281390
end;
13291391
false ->
1330-
?config(rabbitmqctl_cmd, Config)
1392+
case UseSecondaryDist of
1393+
true ->
1394+
SecondaryDist = ?config(secondary_dist, Config),
1395+
rabbit_misc:format(
1396+
"~ts/sbin/rabbitmqctl", [SecondaryDist]);
1397+
false ->
1398+
?config(rabbitmqctl_cmd, Config)
1399+
end
13311400
end,
13321401

13331402
NodeConfig = get_node_config(Config, Node),

deps/rabbitmq_ct_helpers/src/rabbit_ct_helpers.erl

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,7 @@ run_setup_steps(Config, ExtraSteps) ->
7878
[
7979
fun init_skip_as_error_flag/1,
8080
fun guess_tested_erlang_app_name/1,
81+
fun ensure_secondary_dist/1,
8182
fun ensure_secondary_umbrella/1,
8283
fun ensure_current_srcdir/1,
8384
fun ensure_rabbitmq_ct_helpers_srcdir/1,
@@ -201,6 +202,18 @@ guess_tested_erlang_app_name(Config) ->
201202
set_config(Config, {tested_erlang_app, list_to_atom(AppName)})
202203
end.
203204

205+
ensure_secondary_dist(Config) ->
206+
Path = case get_config(Config, secondary_dist) of
207+
undefined -> os:getenv("SECONDARY_DIST");
208+
P -> P
209+
end,
210+
%% Hard fail if the path is invalid.
211+
case Path =:= false orelse filelib:is_dir(Path) of
212+
true -> ok;
213+
false -> error(secondary_dist_path_invalid)
214+
end,
215+
set_config(Config, {secondary_dist, Path}).
216+
204217
ensure_secondary_umbrella(Config) ->
205218
Path = case get_config(Config, secondary_umbrella) of
206219
undefined -> os:getenv("SECONDARY_UMBRELLA");
@@ -1060,11 +1073,13 @@ convert_to_unicode_binary(Arg) when is_binary(Arg) ->
10601073
Arg.
10611074

10621075
is_mixed_versions() ->
1063-
os:getenv("SECONDARY_UMBRELLA") =/= false
1076+
os:getenv("SECONDARY_DIST") =/= false
1077+
orelse os:getenv("SECONDARY_UMBRELLA") =/= false
10641078
orelse os:getenv("RABBITMQ_RUN_SECONDARY") =/= false.
10651079

10661080
is_mixed_versions(Config) ->
1067-
get_config(Config, secondary_umbrella, false) =/= false
1081+
get_config(Config, secondary_dist, false) =/= false
1082+
orelse get_config(Config, secondary_umbrella, false) =/= false
10681083
orelse get_config(Config, rabbitmq_run_secondary_cmd, false) =/= false.
10691084

10701085
%% -------------------------------------------------------------------

0 commit comments

Comments
 (0)