rabbit_feature_flags: Introduce hard vs. soft required feature flags

[Why]
Before this patch, required feature flags were basically checked during
boot: they must have been enabled when they were mere stable feature
flags. If they were not, the node refused to boot.

This was easy for the developer because making a feature flag required
allowed to remove the entire compatibility code. Very satisfying.

Unfortunately, this was a pain point to end users, especially those who
did not pay attention to RabbitMQ and the release notes and were just
asking their package manager to update everything. They could end up
with a node that refuse to boot. The only solution was to downgrade,
enable the disabled stable feature flags, upgrade again.

[How]
This patch introduces two levels of requirement to required feature
flags:
* `hard`: this corresponds to the existing behavior where a node will
  refuse to boot if a hard required feature flag is not enabled before
  the upgrade.
* `soft`: such a required feature flag will be automatically enabled
  during the upgrade to a version where it is marked as required.

The level of requirement is set in the feature flag definition:

    -rabbit_feature_flag(
       {my_feature_flag,
        #{stability     => required,
	  require_level => hard
         }}).

The default requirement level is `soft`. All existing required feature
flags have now a requirement level of `hard`.

The handling of soft required feature flag is done when the cluster
feature flags states are verified and synchronized. If a required
feature flag is not enabled yet, it is enabled at that time.

This means that as developers, we will have to keep compatibility code
forever for every soft required feature flag, like the feature flag
definition itself.

Author: dumbbell

deps/rabbit/src/rabbit_core_ff.erl

@@ -10,81 +10,93 @@
 -rabbit_feature_flag(
    {classic_mirrored_queue_version,
     #{desc          => "Support setting version for classic mirrored queues",
-      stability     => required
+      stability     => required,
+      require_level => hard
      }}).
 
 -rabbit_feature_flag(
    {quorum_queue,
     #{desc          => "Support queues of type `quorum`",
       doc_url       => "https://www.rabbitmq.com/docs/quorum-queues",
-      stability     => required
+      stability     => required,
+      require_level => hard
      }}).
 
 -rabbit_feature_flag(
    {stream_queue,
     #{desc          => "Support queues of type `stream`",
       doc_url       => "https://www.rabbitmq.com/docs/stream",
       stability     => required,
+      require_level => hard,
       depends_on    => [quorum_queue]
      }}).
 
 -rabbit_feature_flag(
    {implicit_default_bindings,
     #{desc          => "Default bindings are now implicit, instead of "
                        "being stored in the database",
-      stability     => required
+      stability     => required,
+      require_level => hard
      }}).
 
 -rabbit_feature_flag(
    {virtual_host_metadata,
     #{desc          => "Virtual host metadata (description, tags, etc)",
-      stability     => required
+      stability     => required,
+      require_level => hard
      }}).
 
 -rabbit_feature_flag(
    {maintenance_mode_status,
     #{desc          => "Maintenance mode status",
-      stability     => required
+      stability     => required,
+      require_level => hard
      }}).
 
 -rabbit_feature_flag(
-    {user_limits,
-     #{desc          => "Configure connection and channel limits for a user",
-       stability     => required
+   {user_limits,
+    #{desc          => "Configure connection and channel limits for a user",
+      stability     => required,
+      require_level => hard
      }}).
 
 -rabbit_feature_flag(
    {stream_single_active_consumer,
     #{desc          => "Single active consumer for streams",
       doc_url       => "https://www.rabbitmq.com/docs/stream",
       stability     => required,
+      require_level => hard,
       depends_on    => [stream_queue]
      }}).
 
 -rabbit_feature_flag(
-    {feature_flags_v2,
-     #{desc          => "Feature flags subsystem V2",
-       stability     => required
+   {feature_flags_v2,
+    #{desc          => "Feature flags subsystem V2",
+      stability     => required,
+      require_level => hard
      }}).
 
 -rabbit_feature_flag(
    {direct_exchange_routing_v2,
-    #{desc       => "v2 direct exchange routing implementation",
-      stability  => required,
-      depends_on => [feature_flags_v2, implicit_default_bindings]
+    #{desc          => "v2 direct exchange routing implementation",
+      stability     => required,
+      require_level => hard,
+      depends_on    => [feature_flags_v2, implicit_default_bindings]
      }}).
 
 -rabbit_feature_flag(
    {listener_records_in_ets,
-    #{desc       => "Store listener records in ETS instead of Mnesia",
-      stability  => required,
-      depends_on => [feature_flags_v2]
+    #{desc          => "Store listener records in ETS instead of Mnesia",
+      stability     => required,
+      require_level => hard,
+      depends_on    => [feature_flags_v2]
      }}).
 
 -rabbit_feature_flag(
    {tracking_records_in_ets,
     #{desc          => "Store tracking records in ETS instead of Mnesia",
       stability     => required,
+      require_level => hard,
       depends_on    => [feature_flags_v2]
      }}).
 
@@ -94,6 +106,7 @@
       doc_url       => "https://github.com/rabbitmq/rabbitmq-server/issues/5931",
       %%TODO remove compatibility code
       stability     => required,
+      require_level => hard,
       depends_on    => [stream_queue]
      }}).
 
@@ -102,6 +115,7 @@
     #{desc          => "Support for restarting streams with optional preferred next leader argument."
       "Used to implement stream leader rebalancing",
       stability     => required,
+      require_level => hard,
       depends_on    => [stream_queue]
      }}).
 
@@ -110,20 +124,23 @@
     #{desc          => "Bug fix to unblock a group of consumers in a super stream partition",
       doc_url       => "https://github.com/rabbitmq/rabbitmq-server/issues/7743",
       stability     => required,
+      require_level => hard,
       depends_on    => [stream_single_active_consumer]
      }}).
 
 -rabbit_feature_flag(
    {stream_filtering,
     #{desc          => "Support for stream filtering.",
       stability     => required,
+      require_level => hard,
       depends_on    => [stream_queue]
      }}).
 
 -rabbit_feature_flag(
    {message_containers,
     #{desc          => "Message containers.",
       stability     => required,
+      require_level => hard,
       depends_on    => [feature_flags_v2]
      }}).
 
@@ -154,6 +171,7 @@
     #{desc          => "A new internal command that is used to update streams as "
                         "part of a policy.",
       stability     => required,
+      require_level => hard,
       depends_on    => [stream_queue]
      }}).
 

deps/rabbit/src/rabbit_feature_flags.erl

@@ -105,6 +105,7 @@
          init/0,
          get_state/1,
          get_stability/1,
+         get_require_level/1,
          check_node_compatibility/1, check_node_compatibility/2,
          sync_feature_flags_with_cluster/2,
          refresh_feature_flags_after_app_load/0,
@@ -147,6 +148,7 @@
 -type feature_props() :: #{desc => string(),
                            doc_url => string(),
                            stability => stability(),
+                           require_level => require_level(),
                            depends_on => [feature_name()],
                            callbacks =>
                            #{callback_name() => callback_fun_name()}}.
@@ -183,6 +185,7 @@
                                     desc => string(),
                                     doc_url => string(),
                                     stability => stability(),
+                                    require_level => require_level(),
                                     depends_on => [feature_name()],
                                     callbacks =>
                                     #{callback_name() => callback_fun_name()},
@@ -207,6 +210,15 @@
 %% Experimental feature flags are not enabled by default on a fresh RabbitMQ
 %% node. They must be enabled by the user.
 
+-type require_level() :: hard | soft.
+%% The level of requirement of a feature flag.
+%%
+%% A hard required feature flags must be enabled before a RabbitMQ node is
+%% upgraded to a version where it is required.
+%%
+%% A soft required feature flag will be automatically enabled when a RabbitMQ
+%% node is upgraded to a version where it is required.
+
 -type callback_fun_name() :: {Module :: module(), Function :: atom()}.
 %% The name of the module and function to call when changing the state of
 %% the feature flag.
@@ -755,6 +767,50 @@ get_stability(FeatureProps) when ?IS_DEPRECATION(FeatureProps) ->
         permitted_by_default -> experimental
     end.
 
+-spec get_require_level
+(FeatureName) -> RequireLevel | undefined when
+      FeatureName :: feature_name(),
+      RequireLevel :: require_level() | none;
+(FeatureProps) -> RequireLevel when
+      FeatureProps ::
+      feature_props_extended() |
+      rabbit_deprecated_features:feature_props_extended(),
+      RequireLevel :: require_level() | none.
+%% @doc
+%% Returns the requirement level of a feature flag.
+%%
+%% The possible requirement levels are:
+%% <ul>
+%% <li>`hard': the feature flag must be enabled before the RabbitMQ node is
+%%   upgraded to a version where it is hard required.</li>
+%% <li>`soft': the feature flag will be automatically enabled wher a RabbitMQ
+%%   node is upgraded to a version where it is soft required.</li>
+%% <li>`none': the feature flag is not required.</li>
+%% </ul>
+%%
+%% @param FeatureName The name of the feature flag to check.
+%% @param FeatureProps A feature flag properties map.
+%% @returns `hard', `soft' or `none', or `undefined' if the given feature flag
+%% name doesn't correspond to a known feature flag.
+
+get_require_level(FeatureName) when is_atom(FeatureName) ->
+    case rabbit_ff_registry_wrapper:get(FeatureName) of
+        undefined    -> undefined;
+        FeatureProps -> get_require_level(FeatureProps)
+    end;
+get_require_level(FeatureProps) when ?IS_FEATURE_FLAG(FeatureProps) ->
+    case get_stability(FeatureProps) of
+        required  -> maps:get(require_level, FeatureProps, soft);
+        undefined -> undefined;
+        _         -> none
+    end;
+get_require_level(FeatureProps) when ?IS_DEPRECATION(FeatureProps) ->
+    case get_stability(FeatureProps) of
+        required -> hard;
+        undefind -> undefined;
+        _        -> none
+    end.
+
 %% -------------------------------------------------------------------
 %% Feature flags registry.
 %% -------------------------------------------------------------------
@@ -913,6 +969,7 @@ assert_feature_flag_is_valid(FeatureName, FeatureProps) ->
                 ValidProps = [desc,
                               doc_url,
                               stability,
+                              require_level,
                               depends_on,
                               callbacks],
                 ?assertEqual([], maps:keys(FeatureProps) -- ValidProps),
@@ -1363,7 +1420,7 @@ run_feature_flags_mod_on_remote_node(Node, Function, Args, Timeout) ->
 sync_feature_flags_with_cluster([] = _Nodes, true = _NodeIsVirgin) ->
     rabbit_ff_controller:enable_default();
 sync_feature_flags_with_cluster([] = _Nodes, false = _NodeIsVirgin) ->
-    ok;
+    rabbit_ff_controller:enable_required();
 sync_feature_flags_with_cluster(Nodes, _NodeIsVirgin) ->
     %% We don't use `rabbit_nodes:filter_running()' here because the given
     %% `Nodes' list may contain nodes which are not members yet (the cluster