WIP

Author: lhoguin

.github/workflows/test-make.yaml

@@ -104,7 +104,7 @@ jobs:
 
     - name: RUN TESTS
       run: |
-        make -C deps/rabbit ct-cluster_upgrade RABBITMQ_RUN_SECONDARY=rabbitmq-server-4.0.2/
+        make -C deps/rabbit ct-cluster_upgrade SECONDARY_DIST=rabbitmq-server-4.0.2/
 
     - name: UPLOAD TEST LOGS
       if: always()

deps/rabbit/test/cluster_upgrade_SUITE.erl

@@ -55,7 +55,7 @@ init_per_testcase(Testcase, Config) ->
     Config1 = rabbit_ct_helpers:set_config(Config, [
         {rmq_nodename_suffix, Testcase},
         {rmq_nodes_count, 3},
-        {force_secondary_umbrella, true}
+        {force_secondary, true}
       ]),
     Config2 = rabbit_ct_helpers:run_steps(Config1,
                 rabbit_ct_broker_helpers:setup_steps() ++
@@ -139,7 +139,7 @@ upgrade_cluster(Config) ->
      || N <- Cluster],
     ct:pal(?LOW_IMPORTANCE, "Restarting cluster ~p", [Cluster]),
     Config1 = rabbit_ct_helpers:set_config(
-                Config, {force_secondary_umbrella, false}),
+                Config, {force_secondary, false}),
     [ok = rabbit_ct_broker_helpers:async_start_node(Config1, N)
      || N <- Cluster],
     [ok = rabbit_ct_broker_helpers:wait_for_async_start_node(N)

deps/rabbit_common/mk/rabbitmq-dist.mk

@@ -1,8 +1,8 @@
 .PHONY: dist test-dist do-dist cli-scripts cli-escripts clean-dist
 
-DIST_DIR = plugins
-CLI_SCRIPTS_DIR = sbin
-CLI_ESCRIPTS_DIR = escript
+DIST_DIR ?= $(CURDIR)/plugins
+CLI_SCRIPTS_DIR ?= $(CURDIR)/sbin
+CLI_ESCRIPTS_DIR ?= $(CURDIR)/escript
 MIX = echo y | mix
 
 # Set $(DIST_AS_EZS) to a non-empty value to enable the packaging of

deps/rabbit_common/mk/rabbitmq-run.mk

@@ -19,7 +19,7 @@ TEST_TMPDIR ?= $(TMPDIR)/rabbitmq-test-instances
 endif
 
 # Location of the scripts controlling the broker.
-RABBITMQ_SCRIPTS_DIR ?= $(CURDIR)/sbin
+RABBITMQ_SCRIPTS_DIR ?= $(CLI_SCRIPTS_DIR)
 
 ifeq ($(PLATFORM),msys2)
 RABBITMQ_PLUGINS ?= $(RABBITMQ_SCRIPTS_DIR)/rabbitmq-plugins.bat
@@ -39,7 +39,7 @@ export RABBITMQ_SCRIPTS_DIR RABBITMQCTL RABBITMQ_PLUGINS RABBITMQ_SERVER RABBITM
 export MAKE
 
 # We need to pass the location of codegen to the Java client ant
-# process.
+# process. @todo Delete?
 CODEGEN_DIR = $(DEPS_DIR)/rabbitmq_codegen
 PYTHONPATH = $(CODEGEN_DIR)
 export PYTHONPATH
@@ -90,7 +90,7 @@ ifdef PLUGINS_FROM_DEPS_DIR
 RMQ_PLUGINS_DIR = $(DEPS_DIR)
 DIST_ERL_LIBS = $(ERL_LIBS)
 else
-RMQ_PLUGINS_DIR = $(CURDIR)/$(DIST_DIR)
+RMQ_PLUGINS_DIR = $(DIST_DIR)
 # We do not want to add apps/ or deps/ to ERL_LIBS
 # when running the release from dist. The `plugins`
 # directory is added to ERL_LIBS by rabbitmq-env.

deps/rabbitmq_ct_helpers/src/rabbit_ct_broker_helpers.erl

@@ -634,13 +634,17 @@ do_start_rabbitmq_node(Config, NodeConfig, I) ->
         false -> WithPlugins0
     end,
     ForceUseSecondary = rabbit_ct_helpers:get_config(
-                          Config, force_secondary_umbrella, undefined),
+                          Config, force_secondary, undefined),
     CanUseSecondary = case ForceUseSecondary of
                           undefined ->
                               (I + 1) rem 2 =:= 0;
                           Override when is_boolean(Override) ->
                               Override
                       end,
+    UseSecondaryDist = case ?config(secondary_dist, Config) of
+                               false -> false;
+                               _     -> CanUseSecondary
+                           end,
     UseSecondaryUmbrella = case ?config(secondary_umbrella, Config) of
                                false -> false;
                                _     -> CanUseSecondary
@@ -686,8 +690,15 @@ do_start_rabbitmq_node(Config, NodeConfig, I) ->
     StartWithPluginsDisabled = rabbit_ct_helpers:get_config(
                                  Config, start_rmq_with_plugins_disabled),
     ExtraArgs2 = case StartWithPluginsDisabled of
-                     true -> ["LEAVE_PLUGINS_DISABLED=yes" | ExtraArgs1];
-                     _    -> ExtraArgs1
+                     true ->
+                        ["LEAVE_PLUGINS_DISABLED=1" | ExtraArgs1];
+                     _ ->
+                      case filename:basename(SrcDir) of
+                          "rabbit" ->
+                              ["LEAVE_PLUGINS_DISABLED=1" | ExtraArgs1];
+                          SrcPlugin ->
+                              [{"RABBITMQ_ENABLED_PLUGINS=~ts", [SrcPlugin]} | ExtraArgs1]
+                      end
                  end,
     KeepPidFile = rabbit_ct_helpers:get_config(
                     Config, keep_pid_file_on_exit),
@@ -731,7 +742,18 @@ do_start_rabbitmq_node(Config, NodeConfig, I) ->
                          {"RABBITMQ_PLUGINS=~ts/rabbitmq-plugins", [SecScriptsDir]}
                          | ExtraArgs4];
                     false ->
-                        ExtraArgs4
+                        case UseSecondaryDist of
+                            true ->
+                                SecondaryDist = ?config(secondary_dist, Config),
+                                [{"DIST_DIR=~ts/plugins", [SecondaryDist]},
+                                 {"CLI_SCRIPTS_DIR=~ts/sbin", [SecondaryDist]},
+                                 {"CLI_ESCRIPTS_DIR=~ts/escript", [SecondaryDist]},
+                                 {"RABBITMQ_SCRIPTS_DIR=~ts/sbin", [SecondaryDist]},
+                                 {"RABBITMQ_SERVER=~ts/sbin/rabbitmq-server", [SecondaryDist]}
+                                | ExtraArgs4];
+                            false ->
+                                ExtraArgs4
+                        end
                 end,
     MakeVars = [
       {"RABBITMQ_NODENAME=~ts", [Nodename]},
@@ -1285,6 +1307,10 @@ rabbitmqctl(Config, Node, Args, Timeout) ->
     CanUseSecondary = (I + 1) rem 2 =:= 0,
     BazelRunSecCmd = rabbit_ct_helpers:get_config(
                        Config, rabbitmq_run_secondary_cmd),
+    UseSecondaryDist = case ?config(secondary_dist, Config) of
+                               false -> false;
+                               _     -> CanUseSecondary
+                           end,
     UseSecondaryUmbrella = case ?config(secondary_umbrella, Config) of
                                false ->
                                    case BazelRunSecCmd of
@@ -1327,7 +1353,14 @@ rabbitmqctl(Config, Node, Args, Timeout) ->
                                      "rabbitmqctl"])
                           end;
                       false ->
-                          ?config(rabbitmqctl_cmd, Config)
+                          case UseSecondaryDist of
+                              true ->
+                                  SecondaryDist = ?config(secondary_dist, Config),
+                                  rabbit_misc:format(
+                                    "~ts/sbin/rabbitmqctl", [SecondaryDist]);
+                              false ->
+                                  ?config(rabbitmqctl_cmd, Config)
+                          end
                   end,
 
     NodeConfig = get_node_config(Config, Node),

deps/rabbitmq_ct_helpers/src/rabbit_ct_helpers.erl

@@ -78,6 +78,7 @@ run_setup_steps(Config, ExtraSteps) ->
             [
                 fun init_skip_as_error_flag/1,
                 fun guess_tested_erlang_app_name/1,
+                fun ensure_secondary_dist/1,
                 fun ensure_secondary_umbrella/1,
                 fun ensure_current_srcdir/1,
                 fun ensure_rabbitmq_ct_helpers_srcdir/1,
@@ -201,6 +202,16 @@ guess_tested_erlang_app_name(Config) ->
             set_config(Config, {tested_erlang_app, list_to_atom(AppName)})
     end.
 
+ensure_secondary_dist(Config) ->
+    Path = case get_config(Config, secondary_dist) of
+               undefined -> os:getenv("SECONDARY_DIST");
+               P         -> P
+           end,
+    case Path =/= false andalso filelib:is_dir(Path) of
+        true  -> set_config(Config, {secondary_dist, Path});
+        false -> set_config(Config, {secondary_dist, false})
+    end.
+
 ensure_secondary_umbrella(Config) ->
     Path = case get_config(Config, secondary_umbrella) of
                undefined -> os:getenv("SECONDARY_UMBRELLA");
@@ -1060,11 +1071,13 @@ convert_to_unicode_binary(Arg) when is_binary(Arg) ->
     Arg.
 
 is_mixed_versions() ->
-    os:getenv("SECONDARY_UMBRELLA") =/= false
+    os:getenv("SECONDARY_DIST") =/= false
+        orelse os:getenv("SECONDARY_UMBRELLA") =/= false
         orelse os:getenv("RABBITMQ_RUN_SECONDARY") =/= false.
 
 is_mixed_versions(Config) ->
-    get_config(Config, secondary_umbrella, false) =/= false
+    get_config(Config, secondary_dist, false) =/= false
+        orelse get_config(Config, secondary_umbrella, false) =/= false
         orelse get_config(Config, rabbitmq_run_secondary_cmd, false) =/= false.
 
 %% -------------------------------------------------------------------