Merge pull request #9005 from rabbitmq/replicate-rabbit_node_maintenance_states-table

rabbit_maintenance: Replicate `rabbit_node_maintenance_states` Mnesia table

Author: dumbbell

deps/rabbit/src/rabbit_db_maintenance.erl

@@ -24,7 +24,7 @@
 %% setup_schema().
 %% -------------------------------------------------------------------
 
--spec setup_schema() -> ok.
+-spec setup_schema() -> ok | {error, any()}.
 %% @doc Creates the internal schema used by the selected metadata store
 %%
 %% @private
@@ -40,9 +40,41 @@ setup_schema_in_mnesia() ->
       "Creating table ~ts for maintenance mode status",
       [TableName]),
     try
-        _ = rabbit_table:create(
-              TableName,
-              status_table_definition())
+        rabbit_table:create(
+          TableName,
+          status_table_definition()),
+        %% The `rabbit_node_maintenance_states' table used to be global but not
+        %% replicated. This leads to various errors during RabbitMQ boot or
+        %% operations on the Mnesia database. The reason is the table existed
+        %% on a single node and, if that node was stopped or MIA, other nodes
+        %% may wait forever on that node for the table to be available.
+        %%
+        %% The call below makes sure this node has a copy of the table.
+        case rabbit_table:ensure_table_copy(TableName, node(), ram_copies) of
+            ok ->
+                %% Next, we try to fix other nodes in the cluster if they are
+                %% running a version of RabbitMQ which does not replicate the
+                %% table. All nodes must have a replica for Mnesia operations
+                %% to work properly. Therefore the code below is to make older
+                %% compatible with newer nodes.
+                Replicas = mnesia:table_info(TableName, all_nodes),
+                Members = rabbit_nodes:list_running(),
+                MissingOn = Members -- Replicas,
+                lists:foreach(
+                  fun(Node) ->
+                          %% Errors from adding a replica on those older nodes
+                          %% are ignored however. They should not be fatal. The
+                          %% problem will solve by itself once all nodes are
+                          %% upgraded.
+                          _ = rpc:call(
+                                Node,
+                                rabbit_table, ensure_table_copy,
+                                [TableName, Node, ram_copies])
+                  end, MissingOn),
+                ok;
+            Error ->
+                Error
+        end
     catch throw:Reason  ->
             rabbit_log:error(
               "Failed to create maintenance status table: ~tp",