Feature flags: Handle plugins' feature flags only present on some nodes

Before this patch, when a node had a plugin which provided a feature
flag, that node could not join a cluster because the other nodes didn't
know about that plugin's feature flag. This was the same problem when an
operator enabled a plugin present on some nodes (but not all) in a
cluster and then wanted to enable the plugin's feature flags.

This situation is fixed by paying attention to the Erlang applications
(plugins) providing each feature flag, when we want to determine if two
nodes are compatible.

To achieve this and still maintain the same view & state on all nodes,
when a node (re)joins a cluster, all feature flags from both sides are
exchanged: if a feature flag on one side is provided by an application
which is missing on the other side, that feature flag is added to the
latter's registry.

After this exchange, we proceed with the regular compatibility check.
Therefore, feature flags provided by unknown applications are supported
everywhere and thus won't interfere.

Also in this patch is a fix of the registry generation: the way feature
flag states were handled was incorrect: reinitializing the registry
could loose states because `initialize_registry/3` would take the
complete list of enabled feature flags. Now it is transformed to take a
"diff": a map indicating which feature flags are enabled/disabled or
marked as `state_changing`. We now store a map of those states inside
the registry.

One change of behavior with this patch is: feature flags are enabled by
default only if it is a virgin node (it is the first time it starts or
it was reset), even if it is a single non-clustered node.

Finally, the testsuite for feature flags was expanded to cover
various clustering and plugin activation situations.

[#163796129]

Author: dumbbell

src/rabbit.erl

@@ -520,7 +520,7 @@ start_apps(Apps) ->
 
 start_apps(Apps, RestartTypes) ->
     app_utils:load_applications(Apps),
-    rabbit_feature_flags:initialize_registry(),
+    ok = rabbit_feature_flags:refresh_feature_flags_after_app_load(Apps),
     start_loaded_apps(Apps, RestartTypes).
 
 start_loaded_apps(Apps) ->

src/rabbit_feature_flags.erl

@@ -31,6 +31,7 @@
 
 -export([get/1,
          list/1,
+         states/0,
          is_supported/1,
          is_enabled/1,
          is_registry_initialized/0,
@@ -76,6 +77,23 @@ list(Which) ->
         true  -> #{}
     end.
 
+-spec states() -> rabbit_feature_flags:feature_states().
+%% @doc
+%% Returns the states of supported feature flags.
+%%
+%% Only the informations stored in the local registry is used to answer
+%% this call.
+%%
+%% @returns A map of feature flag states.
+
+states() ->
+    rabbit_feature_flags:initialize_registry(),
+    %% See get/1 for an explanation of the case statement below.
+    case is_registry_initialized() of
+        false -> ?MODULE:states();
+        true  -> #{}
+    end.
+
 -spec is_supported(rabbit_feature_flags:feature_name()) -> boolean().
 %% @doc
 %% Returns if a feature flag is supported.

src/rabbit_ff_registry.erl

@@ -528,6 +528,7 @@ dir() -> mnesia:system_info(directory).
 %% nodes in the cluster already. It also updates the cluster status
 %% file.
 init_db(ClusterNodes, NodeType, CheckOtherNodes) ->
+    NodeIsVirgin = is_virgin_node(),
     Nodes = change_extra_db_nodes(ClusterNodes, CheckOtherNodes),
     %% Note that we use `system_info' here and not the cluster status
     %% since when we start rabbit for the first time the cluster
@@ -551,7 +552,7 @@ init_db(ClusterNodes, NodeType, CheckOtherNodes) ->
             ok = rabbit_table:wait_for_replicated(_Retry = true),
             ok = rabbit_table:create_local_copy(NodeType)
     end,
-    ensure_feature_flags_are_in_sync(Nodes),
+    ensure_feature_flags_are_in_sync(Nodes, NodeIsVirgin),
     ensure_schema_integrity(),
     rabbit_node_monitor:update_cluster_status(),
     ok.
@@ -621,12 +622,12 @@ ensure_mnesia_not_running() ->
             throw({error, mnesia_unexpectedly_running})
     end.
 
-ensure_feature_flags_are_in_sync(Nodes) ->
-    case rabbit_feature_flags:sync_feature_flags_with_cluster(Nodes) of
-        ok ->
-            ok;
-        {error, Reason} ->
-            throw({error, {incompatible_feature_flags, Reason}})
+ensure_feature_flags_are_in_sync(Nodes, NodeIsVirgin) ->
+    Ret = rabbit_feature_flags:sync_feature_flags_with_cluster(
+            Nodes, NodeIsVirgin),
+    case Ret of
+        ok              -> ok;
+        {error, Reason} -> throw({error, {incompatible_feature_flags, Reason}})
     end.
 
 ensure_schema_integrity() ->

src/rabbit_mnesia.erl

@@ -469,7 +469,6 @@ clean_plugins(Plugins) ->
 clean_plugin(Plugin, ExpandDir) ->
     {ok, Mods} = application:get_key(Plugin, modules),
     application:unload(Plugin),
-    rabbit_feature_flags:initialize_registry(),
     [begin
          code:soft_purge(Mod),
          code:delete(Mod),
@@ -714,4 +713,6 @@ remove_plugins(Plugins) ->
 maybe_report_plugin_loading_problems([]) ->
     ok;
 maybe_report_plugin_loading_problems(Problems) ->
-    rabbit_log:warning("Problem reading some plugins: ~p~n", [Problems]).
+    io:format(standard_error,
+              "Problem reading some plugins: ~p~n",
+              [Problems]).