Merge pull request #2029 from rabbitmq/v3.7.x-backport-feature-flags-subsystem

Backport feature flags subsystem to RabbitMQ 3.7.x

Author: michaelklishin

Makefile

@@ -127,9 +127,9 @@ define PROJECT_ENV
 	  ]
 endef
 
-LOCAL_DEPS = sasl mnesia os_mon inets
+LOCAL_DEPS = sasl mnesia os_mon inets compiler syntax_tools
 BUILD_DEPS = rabbitmq_cli syslog
-DEPS = ranch lager rabbit_common sysmon_handler recon observer_cli
+DEPS = ranch lager rabbit_common sysmon_handler recon observer_cli stdout_formatter
 TEST_DEPS = rabbitmq_ct_helpers rabbitmq_ct_client_helpers amqp_client meck proper
 
 dep_syslog = git https://github.com/schlagert/syslog 3.4.5
@@ -182,6 +182,7 @@ SLOW_CT_SUITES := backing_queue \
 		  config_schema \
 		  dynamic_ha \
 		  eager_sync \
+		  feature_flags \
 		  health_check \
 		  lazy_queue \
 		  metrics \

scripts/rabbitmq-env

@@ -276,6 +276,10 @@ rmq_normalize_path_var RABBITMQ_PID_FILE
 
 [ "x" = "x$RABBITMQ_BOOT_MODULE" ] && RABBITMQ_BOOT_MODULE=${BOOT_MODULE}
 
+[ "x" != "x$RABBITMQ_FEATURE_FLAGS_FILE" ] && RABBITMQ_FEATURE_FLAGS_FILE_source=environment
+[ "x" = "x$RABBITMQ_FEATURE_FLAGS_FILE" ] && RABBITMQ_FEATURE_FLAGS_FILE=${RABBITMQ_MNESIA_BASE}/${RABBITMQ_NODENAME}-feature_flags
+rmq_normalize_path_var RABBITMQ_FEATURE_FLAGS_FILE
+
 [ "x" = "x$RABBITMQ_PLUGINS_EXPAND_DIR" ] && RABBITMQ_PLUGINS_EXPAND_DIR=${PLUGINS_EXPAND_DIR}
 [ "x" = "x$RABBITMQ_PLUGINS_EXPAND_DIR" ] && RABBITMQ_PLUGINS_EXPAND_DIR=${RABBITMQ_MNESIA_BASE}/${RABBITMQ_NODENAME}-plugins-expand
 rmq_normalize_path_var RABBITMQ_PLUGINS_EXPAND_DIR
@@ -308,6 +312,7 @@ rmq_check_if_shared_with_mnesia \
     RABBITMQ_CONFIG_FILE \
     RABBITMQ_LOG_BASE \
     RABBITMQ_PID_FILE \
+    RABBITMQ_FEATURE_FLAGS_FILE \
     RABBITMQ_PLUGINS_EXPAND_DIR \
     RABBITMQ_ENABLED_PLUGINS_FILE \
     RABBITMQ_PLUGINS_DIR \
@@ -318,7 +323,8 @@ rmq_check_if_shared_with_mnesia \
 
 ## Development-specific environment.
 if [ "${RABBITMQ_DEV_ENV}" ]; then
-    if [ "$RABBITMQ_PLUGINS_DIR_source" != 'environment' -o \
+    if [ "$RABBITMQ_FEATURE_FLAGS_FILE_source" != 'environment' -o \
+         "$RABBITMQ_PLUGINS_DIR_source" != 'environment' -o \
          "$RABBITMQ_ENABLED_PLUGINS_FILE_source" != 'environment' ]; then
         # We need to query the running node for the plugins directory
         # and the "enabled plugins" file.
@@ -338,16 +344,23 @@ if [ "${RABBITMQ_DEV_ENV}" ]; then
           esac
         done
         eval $( (${RABBITMQ_SCRIPTS_DIR}/rabbitmqctl $node_arg eval \
-            '{ok, P} = application:get_env(rabbit, plugins_dir),
+            '{ok, F} = application:get_env(rabbit, feature_flags_file),
+             {ok, P} = application:get_env(rabbit, plugins_dir),
              {ok, E} = application:get_env(rabbit, enabled_plugins_file),
              B = os:getenv("RABBITMQ_MNESIA_BASE"),
              M = os:getenv("RABBITMQ_MNESIA_DIR"),
              io:format(
+               "feature_flags_file=\"~s\"~n"
                "plugins_dir=\"~s\"~n"
                "enabled_plugins_file=\"~s\"~n"
                "mnesia_base=\"~s\"~n"
-               "mnesia_dir=\"~s\"~n", [P, E, B, M]).' \
-            2>/dev/null | grep -E '^(plugins_dir|enabled_plugins_file|mnesia_base|mnesia_dir)=') || :)
+               "mnesia_dir=\"~s\"~n", [F, P, E, B, M]).' \
+            2>/dev/null | grep -E '^(feature_flags_file|plugins_dir|enabled_plugins_file|mnesia_base|mnesia_dir)=') || :)
+
+        if [ "${feature_flags_file}" -a \
+             "$RABBITMQ_FEATURE_FLAGS_FILE_source" != 'environment' ]; then
+            RABBITMQ_FEATURE_FLAGS_FILE="${feature_flags_file}"
+        fi
         if [ "${plugins_dir}" -a \
              "$RABBITMQ_PLUGINS_DIR_source" != 'environment' ]; then
             RABBITMQ_PLUGINS_DIR="${plugins_dir}"

scripts/rabbitmq-env.bat

@@ -275,6 +275,15 @@ if "!RABBITMQ_BOOT_MODULE!"=="" (
     )
 )
 
+REM [ "x" = "x$RABBITMQ_FEATURE_FLAGS_FILE" ] && RABBITMQ_FEATURE_FLAGS_FILE=${RABBITMQ_MNESIA_BASE}/${RABBITMQ_NODENAME}-feature_flags
+if "!RABBITMQ_FEATURE_FLAGS_FILE!"=="" (
+    if "!FEATURE_FLAGS_FILE!"=="" (
+        set RABBITMQ_FEATURE_FLAGS_FILE=!RABBITMQ_MNESIA_BASE!\!RABBITMQ_NODENAME!-feature_flags
+    ) else (
+        set RABBITMQ_FEATURE_FLAGS_FILE=!FEATURE_FLAGS_FILE!
+    )
+)
+
 REM [ "x" = "x$RABBITMQ_PLUGINS_EXPAND_DIR" ] && RABBITMQ_PLUGINS_EXPAND_DIR=${PLUGINS_EXPAND_DIR}
 REM [ "x" = "x$RABBITMQ_PLUGINS_EXPAND_DIR" ] && RABBITMQ_PLUGINS_EXPAND_DIR=${RABBITMQ_MNESIA_BASE}/${RABBITMQ_NODENAME}-plugins-expand
 if "!RABBITMQ_PLUGINS_EXPAND_DIR!"=="" (
@@ -388,6 +397,13 @@ if defined RABBITMQ_DEV_ENV (
     if "!SCRIPT_NAME!" == "rabbitmq-plugins" (
         REM We may need to query the running node for the plugins directory
         REM and the "enabled plugins" file.
+        if not "%RABBITMQ_FEATURE_FLAGS_FILE_source%" == "environment" (
+            for /f "delims=" %%F in ('!SCRIPT_DIR!\rabbitmqctl eval "{ok, P} = application:get_env(rabbit, feature_flags_file), io:format(""~s~n"", [P])."') do @set feature_flags_file=%%F
+            if exist "!feature_flags_file!" (
+                set RABBITMQ_FEATURE_FLAGS_FILE=!feature_flags_file!
+            )
+            REM set feature_flags_file=
+        )
         if not "%RABBITMQ_PLUGINS_DIR_source%" == "environment" (
             for /f "delims=" %%F in ('!SCRIPT_DIR!\rabbitmqctl eval "{ok, P} = application:get_env(rabbit, plugins_dir), io:format(""~s~n"", [P])."') do @set plugins_dir=%%F
             if exist "!plugins_dir!" (
@@ -467,6 +483,7 @@ exit /b
 REM Environment cleanup
 set BOOT_MODULE=
 set CONFIG_FILE=
+set FEATURE_FLAGS_FILE=
 set ENABLED_PLUGINS_FILE=
 set LOG_BASE=
 set MNESIA_BASE=

scripts/rabbitmq-server

@@ -197,6 +197,7 @@ RABBITMQ_DIST_PORT=$RABBITMQ_DIST_PORT \
     ${RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS} \
     ${RABBITMQ_NAME_TYPE} ${RABBITMQ_PRELAUNCH_NODENAME} \
     -conf_advanced "${RABBITMQ_ADVANCED_CONFIG_FILE}" \
+    -rabbit feature_flags_file "\"$RABBITMQ_FEATURE_FLAGS_FILE\"" \
     -rabbit enabled_plugins_file "\"$RABBITMQ_ENABLED_PLUGINS_FILE\"" \
     -rabbit plugins_dir "\"$RABBITMQ_PLUGINS_DIR\"" \
     -extra "${RABBITMQ_NODENAME}"
@@ -304,6 +305,7 @@ start_rabbitmq_server() {
         -rabbit lager_log_root "\"$RABBITMQ_LOG_BASE\"" \
         -rabbit lager_default_file "$RABBIT_LAGER_HANDLER" \
         -rabbit lager_upgrade_file "$RABBITMQ_LAGER_HANDLER_UPGRADE" \
+        -rabbit feature_flags_file "\"$RABBITMQ_FEATURE_FLAGS_FILE\"" \
         -rabbit enabled_plugins_file "\"$RABBITMQ_ENABLED_PLUGINS_FILE\"" \
         -rabbit plugins_dir "\"$RABBITMQ_PLUGINS_DIR\"" \
         -rabbit plugins_expand_dir "\"$RABBITMQ_PLUGINS_EXPAND_DIR\"" \

scripts/rabbitmq-server.bat

@@ -149,6 +149,7 @@ if "!RABBITMQ_CONFIG_FILE_NOEX!.conf" == "!RABBITMQ_CONFIG_FILE!" (
         -s rabbit_prelaunch ^
         !RABBITMQ_NAME_TYPE! rabbitmqprelaunch!RANDOM!!TIME:~9!@localhost ^
         -conf_advanced "!RABBITMQ_ADVANCED_CONFIG_FILE!" ^
+        -rabbit feature_flags_file "!RABBITMQ_FEATURE_FLAGS_FILE!" ^
         -rabbit enabled_plugins_file "!RABBITMQ_ENABLED_PLUGINS_FILE!" ^
         -rabbit plugins_dir "!RABBITMQ_PLUGINS_DIR!" ^
         -extra "!RABBITMQ_NODENAME!"
@@ -250,6 +251,7 @@ if "!ENV_OK!"=="false" (
 -rabbit lager_log_root \""!RABBITMQ_LOG_BASE:\=/!"\" ^
 -rabbit lager_default_file !RABBIT_LAGER_HANDLER! ^
 -rabbit lager_upgrade_file !RABBITMQ_LAGER_HANDLER_UPGRADE! ^
+-rabbit feature_flags_file \""!RABBITMQ_FEATURE_FLAGS_FILE:\=/!"\" ^
 -rabbit enabled_plugins_file \""!RABBITMQ_ENABLED_PLUGINS_FILE:\=/!"\" ^
 -rabbit plugins_dir \""!RABBITMQ_PLUGINS_DIR:\=/!"\" ^
 -rabbit plugins_expand_dir \""!RABBITMQ_PLUGINS_EXPAND_DIR:\=/!"\" ^

scripts/rabbitmq-service.bat

@@ -232,6 +232,7 @@ if "!RABBITMQ_CONFIG_FILE_NOEX!.conf" == "!RABBITMQ_CONFIG_FILE!" (
         -s rabbit_prelaunch ^
         !RABBITMQ_NAME_TYPE! rabbitmqprelaunch!RANDOM!!TIME:~9!@localhost ^
         -conf_advanced "!RABBITMQ_ADVANCED_CONFIG_FILE!" ^
+        -rabbit feature_flags_file "!RABBITMQ_FEATURE_FLAGS_FILE!" ^
         -rabbit enabled_plugins_file "!RABBITMQ_ENABLED_PLUGINS_FILE!" ^
         -rabbit plugins_dir "!RABBITMQ_PLUGINS_DIR!" ^
         -extra "!RABBITMQ_NODENAME!"
@@ -323,6 +324,7 @@ set ERLANG_SERVICE_ARGUMENTS= ^
 -rabbit lager_log_root \""!RABBITMQ_LOG_BASE:\=/!"\" ^
 -rabbit lager_default_file !RABBIT_LAGER_HANDLER! ^
 -rabbit lager_upgrade_file !RABBITMQ_LAGER_HANDLER_UPGRADE! ^
+-rabbit feature_flags_file \""!RABBITMQ_FEATURE_FLAGS_FILE:\=/!"\" ^
 -rabbit enabled_plugins_file \""!RABBITMQ_ENABLED_PLUGINS_FILE:\=/!"\" ^
 -rabbit plugins_dir \""!RABBITMQ_PLUGINS_DIR:\=/!"\" ^
 -rabbit plugins_expand_dir \""!RABBITMQ_PLUGINS_EXPAND_DIR:\=/!"\" ^

src/rabbit.erl

@@ -63,6 +63,12 @@
                     {requires,    pre_boot},
                     {enables,     external_infrastructure}]}).
 
+-rabbit_boot_step({feature_flags,
+                   [{description, "feature flags registry and initial state"},
+                    {mfa,         {rabbit_feature_flags, init, []}},
+                    {requires,    pre_boot},
+                    {enables,     external_infrastructure}]}).
+
 -rabbit_boot_step({database,
                    [{mfa,         {rabbit_mnesia, init, []}},
                     {requires,    file_handle_cache},
@@ -314,9 +320,11 @@ start() ->
                      HipeResult = rabbit_hipe:maybe_hipe_compile(),
                      ok = start_logger(),
                      rabbit_hipe:log_hipe_result(HipeResult),
+                     Apps = load_all_apps(),
+                     rabbit_feature_flags:initialize_registry(),
                      rabbit_node_monitor:prepare_cluster_status_files(),
                      rabbit_mnesia:check_cluster_consistency(),
-                     broker_start()
+                     broker_start(Apps)
              end).
 
 boot() ->
@@ -326,13 +334,15 @@ boot() ->
                      HipeResult = rabbit_hipe:maybe_hipe_compile(),
                      ok = start_logger(),
                      rabbit_hipe:log_hipe_result(HipeResult),
+                     Apps = load_all_apps(),
+                     rabbit_feature_flags:initialize_registry(),
                      rabbit_node_monitor:prepare_cluster_status_files(),
                      ok = rabbit_upgrade:maybe_upgrade_mnesia(),
                      %% It's important that the consistency check happens after
                      %% the upgrade, since if we are a secondary node the
                      %% primary node will have forgotten us
                      rabbit_mnesia:check_cluster_consistency(),
-                     broker_start()
+                     broker_start(Apps)
              end).
 
 ensure_config() ->
@@ -347,11 +357,14 @@ ensure_config() ->
         ok -> ok
     end.
 
-
-broker_start() ->
+load_all_apps() ->
     Plugins = rabbit_plugins:setup(),
     ToBeLoaded = Plugins ++ ?APPS,
-    start_apps(ToBeLoaded),
+    app_utils:load_applications(ToBeLoaded),
+    ToBeLoaded.
+
+broker_start(Apps) ->
+    start_loaded_apps(Apps),
     maybe_sd_notify(),
     ok = rabbit_lager:broker_is_started(),
     ok = log_broker_started(rabbit_plugins:strictly_plugins(rabbit_plugins:active())).
@@ -533,6 +546,13 @@ start_apps(Apps) ->
 
 start_apps(Apps, RestartTypes) ->
     app_utils:load_applications(Apps),
+    ok = rabbit_feature_flags:refresh_feature_flags_after_app_load(Apps),
+    start_loaded_apps(Apps, RestartTypes).
+
+start_loaded_apps(Apps) ->
+    start_loaded_apps(Apps, #{}).
+
+start_loaded_apps(Apps, RestartTypes) ->
     ensure_sysmon_handler_app_config(),
     ConfigEntryDecoder = case application:get_env(rabbit, config_entry_decoder) of
         undefined ->