Hide maintenance mode behind a feature flag

Otherwise we'd lose mixed version cluster compatibility.
We cannot pre-create a table since it will be checked for existence
on cluster peers.
We cannot use an upgrade function since it assumes a cluster-wide
restart.

A feature flag avoids as many headaches as possible and when
flags are detected to be compatible on node start, they are
all enabled automatically.

References #2321.

5 files changed, 117 insertions, 46 deletions

diff --git a/src/rabbit_core_ff.erl b/src/rabbit_core_ff.erl
index e4c01339a9..7a8cae5e40 100644
--- a/src/rabbit_core_ff.erl
+++ b/src/rabbit_core_ff.erl
@@ -9,7 +9,8 @@
 
 -export([quorum_queue_migration/3,
          implicit_default_bindings_migration/3,
-         virtual_host_metadata_migration/3]).
+         virtual_host_metadata_migration/3,
+         maintenance_mode_status_migration/3]).
 
 -rabbit_feature_flag(
    {quorum_queue,
@@ -34,6 +35,14 @@
       migration_fun => {?MODULE, virtual_host_metadata_migration}
      }}).
 
+-rabbit_feature_flag(
+    {maintenance_mode_status,
+     #{
+         desc          => "Maintenance mode status",
+         stability     => stable,
+         migration_fun => {?MODULE, maintenance_mode_status_migration}
+      }}).
+
 %% -------------------------------------------------------------------
 %% Quorum queues.
 %% -------------------------------------------------------------------
@@ -112,3 +121,21 @@ virtual_host_metadata_migration(_FeatureName, _FeatureProps, enable) ->
     end;
 virtual_host_metadata_migration(_FeatureName, _FeatureProps, is_enabled) ->
     mnesia:table_info(rabbit_vhost, attributes) =:= vhost:fields(vhost_v2).
+
+
+%%
+%% Maintenance mode
+%%
+
+maintenance_mode_status_migration(FeatureName, _FeatureProps, enable) ->
+    TableName = rabbit_maintenance:status_table_name(),
+    rabbit_log:info("Creating table ~s for feature flag `~s`", [TableName, FeatureName]),
+    case rabbit_table:create(TableName, rabbit_maintenance:status_table_definition()) of
+        ok ->
+            rabbit_table:ensure_table_copy(TableName, node());
+        {error, Reason} ->
+            rabbit_log:error("Failed to create maintenance status table: ~p", [Reason]),
+            {error, Reason}
+    end;
+maintenance_mode_status_migration(_FeatureName, _FeatureProps, is_enabled) ->
+    rabbit_table:exists(rabbit_maintenance:status_table_name()).
diff --git a/src/rabbit_maintenance.erl b/src/rabbit_maintenance.erl
index 7ca9a3ce7c..c6e3fbbc38 100644
--- a/src/rabbit_maintenance.erl
+++ b/src/rabbit_maintenance.erl
@@ -19,6 +19,7 @@
 -include("rabbit.hrl").
 
 -export([
+    is_enabled/0,
     drain/0,
     revive/0,
     mark_as_being_drained/0,
@@ -35,9 +36,13 @@
     primary_replica_transfer_candidate_nodes/0,
     random_primary_replica_transfer_candidate_node/1,
     transfer_leadership_of_quorum_queues/1,
-    transfer_leadership_of_classic_mirrored_queues/1]).
+    transfer_leadership_of_classic_mirrored_queues/1,
+    status_table_name/0,
+    status_table_definition/0
+]).
 
 -define(TABLE, rabbit_node_maintenance_states).
+-define(FEATURE_FLAG, maintenance_mode_status).
 -define(DEFAULT_STATUS,  regular).
 -define(DRAINING_STATUS, draining).
 
@@ -51,8 +56,30 @@
 %% API
 %%
 
+-spec status_table_name() -> mnesia:table().
+status_table_name() ->
+    ?TABLE.
+
+-spec status_table_definition() -> list().
+status_table_definition() ->
+    maps:to_list(#{
+        record_name => node_maintenance_state,
+        attributes  => record_info(fields, node_maintenance_state)
+    }).
+
+-spec is_enabled() -> boolean().
+is_enabled() ->
+    rabbit_feature_flags:is_enabled(?FEATURE_FLAG).
+
 -spec drain() -> ok.
 drain() ->
+    case is_enabled() of
+        true  -> do_drain();
+        false -> rabbit_log:warning("Feature flag `~s` is not enabled, draining is a no-op", [?FEATURE_FLAG])
+    end.
+
+-spec do_drain() -> ok.
+do_drain() ->
     rabbit_log:alert("This node is being put into maintenance (drain) mode"),
     mark_as_being_drained(),
     rabbit_log:info("Marked this node as undergoing maintenance"),
@@ -71,7 +98,7 @@ drain() ->
                     [length(TransferCandidates), ReadableCandidates]),
     transfer_leadership_of_classic_mirrored_queues(TransferCandidates),
     transfer_leadership_of_quorum_queues(TransferCandidates),
-    
+
     %% allow plugins to react
     rabbit_event:notify(maintenance_draining, #{
         reason => <<"node is being put into maintenance">>
@@ -82,6 +109,13 @@ drain() ->
 
 -spec revive() -> ok.
 revive() ->
+    case is_enabled() of
+        true  -> do_revive();
+        false -> rabbit_log:warning("Feature flag `~s` is not enabled, reviving is a no-op", [?FEATURE_FLAG])
+    end.
+
+-spec do_revive() -> ok.
+do_revive() ->
     rabbit_log:alert("This node is being revived from maintenance (drain) mode"),
     revive_local_quorum_queue_replicas(),
     rabbit_log:alert("Resumed all listeners and will accept client connections again"),
@@ -92,9 +126,9 @@ revive() ->
 
     %% allow plugins to react
     rabbit_event:notify(maintenance_revived, #{}),
-    
-    ok.
 
+    ok.
+ 
 -spec mark_as_being_drained() -> boolean().
 mark_as_being_drained() ->
     rabbit_log:debug("Marking the node as undergoing maintenance"),
diff --git a/src/rabbit_mnesia.erl b/src/rabbit_mnesia.erl
index f22e975b82..070c6a8205 100644
--- a/src/rabbit_mnesia.erl
+++ b/src/rabbit_mnesia.erl
@@ -571,7 +571,7 @@ init_db(ClusterNodes, NodeType, CheckOtherNodes) ->
             %% Subsequent node in cluster, catch up
             maybe_force_load(),
             ok = rabbit_table:wait_for_replicated(_Retry = true),
-            ok = rabbit_table:create_local_copy(NodeType)
+            ok = rabbit_table:ensure_local_copies(NodeType)
     end,
     ensure_feature_flags_are_in_sync(Nodes, NodeIsVirgin),
     ensure_schema_integrity(),
diff --git a/src/rabbit_table.erl b/src/rabbit_table.erl
index a922190647..05f9a8d381 100644
--- a/src/rabbit_table.erl
+++ b/src/rabbit_table.erl
@@ -7,10 +7,12 @@
 
 -module(rabbit_table).
 
--export([create/0, create_local_copy/1, wait_for_replicated/1, wait/1, wait/2,
-         force_load/0, is_present/0, is_empty/0, needs_default_data/0,
-         check_schema_integrity/1, clear_ram_only_tables/0, retry_timeout/0,
-         wait_for_replicated/0]).
+-export([
+    create/0, create/2, ensure_local_copies/1, ensure_table_copy/2,
+    wait_for_replicated/1, wait/1, wait/2,
+    force_load/0, is_present/0, is_empty/0, needs_default_data/0,
+    check_schema_integrity/1, clear_ram_only_tables/0, retry_timeout/0,
+    wait_for_replicated/0, exists/1]).
 
 %% for testing purposes
 -export([definitions/0]).
@@ -28,19 +30,29 @@
 -spec create() -> 'ok'.
 
 create() ->
-    lists:foreach(fun ({Tab, TabDef}) ->
-                          TabDef1 = proplists:delete(match, TabDef),
-                          rabbit_log:debug("Will create a schema database table named '~s'", [Tab]),
-                          case mnesia:create_table(Tab, TabDef1) of
-                              {atomic, ok} -> ok;
-                              {aborted, Reason} ->
-                                  throw({error, {table_creation_failed,
-                                                 Tab, TabDef1, Reason}})
-                          end
-                  end, definitions()),
+    lists:foreach(
+        fun ({Table, Def}) -> create(Table, Def) end,
+        definitions()),
     ensure_secondary_indexes(),
     ok.
 
+-spec create(mnesia:table(), list()) -> rabbit_types:ok_or_error(any()).
+
+create(TableName, TableDefinition) ->
+    TableDefinition1 = proplists:delete(match, TableDefinition),
+    rabbit_log:debug("Will create a schema database table '~s'", [TableName]),
+    case mnesia:create_table(TableName, TableDefinition1) of
+        {atomic, ok}                              -> ok;
+        {aborted,{already_exists, TableName}}     -> ok;
+        {aborted, {already_exists, TableName, _}} -> ok;
+        {aborted, Reason}                         ->
+            throw({error, {table_creation_failed, TableName, TableDefinition1, Reason}})
+    end.
+
+-spec exists(mnesia:table()) -> boolean().
+exists(Table) ->
+    lists:member(Table, mnesia:system_info(tables)).
+
 %% Sets up secondary indexes in a blank node database.
 ensure_secondary_indexes() ->
   ensure_secondary_index(rabbit_queue, vhost),
@@ -52,19 +64,15 @@ ensure_secondary_index(Table, Field) ->
     {aborted, {already_exists, Table, _}} -> ok
   end.
 
-%% The sequence in which we delete the schema and then the other
-%% tables is important: if we delete the schema first when moving to
-%% RAM mnesia will loudly complain since it doesn't make much sense to
-%% do that. But when moving to disc, we need to move the schema first.
-
--spec create_local_copy('disc' | 'ram') -> 'ok'.
-
-create_local_copy(disc) ->
-    create_local_copy(schema, disc_copies),
-    create_local_copies(disc);
-create_local_copy(ram)  ->
-    create_local_copies(ram),
-    create_local_copy(schema, ram_copies).
+-spec ensure_table_copy(mnesia:table(), node()) -> ok | {error, any()}.
+ensure_table_copy(TableName, Node) ->
+    rabbit_log:debug("Will add a local schema database copy for table '~s'", [TableName]),
+    case mnesia:add_table_copy(TableName, Node, disc_copies) of
+        {atomic, ok}                              -> ok;
+        {aborted,{already_exists, TableName}}     -> ok;
+        {aborted, {already_exists, TableName, _}} -> ok;
+        {aborted, Reason}                         -> {error, Reason}
+    end.
 
 %% This arity only exists for backwards compatibility with certain
 %% plugins. See https://github.com/rabbitmq/rabbitmq-clusterer/issues/19.
@@ -181,6 +189,20 @@ clear_ram_only_tables() ->
       end, names()),
     ok.
 
+%% The sequence in which we delete the schema and then the other
+%% tables is important: if we delete the schema first when moving to
+%% RAM mnesia will loudly complain since it doesn't make much sense to
+%% do that. But when moving to disc, we need to move the schema first.
+
+-spec ensure_local_copies('disc' | 'ram') -> 'ok'.
+
+ensure_local_copies(disc) ->
+    create_local_copy(schema, disc_copies),
+    create_local_copies(disc);
+ensure_local_copies(ram)  ->
+    create_local_copies(ram),
+    create_local_copy(schema, ram_copies).
+
 %%--------------------------------------------------------------------
 %% Internal helpers
 %%--------------------------------------------------------------------
@@ -364,11 +386,7 @@ definitions() ->
      {rabbit_queue,
       [{record_name, amqqueue},
        {attributes, amqqueue:fields()},
-       {match, amqqueue:pattern_match_on_name(queue_name_match())}]},
-     {rabbit_node_maintenance_states,
-      [{record_name, node_maintenance_state},
-       {attributes, record_info(fields, node_maintenance_state)},
-       {match, #node_maintenance_state{_='_'}}]}
+       {match, amqqueue:pattern_match_on_name(queue_name_match())}]}
     ]
         ++ gm:table_definitions()
         ++ mirrored_supervisor:table_definitions().
diff --git a/src/rabbit_upgrade_functions.erl b/src/rabbit_upgrade_functions.erl
index ae80075e8f..59417c72bb 100644
--- a/src/rabbit_upgrade_functions.erl
+++ b/src/rabbit_upgrade_functions.erl
@@ -52,7 +52,6 @@
 -rabbit_upgrade({topic_permission,      mnesia,  []}).
 -rabbit_upgrade({queue_options,         mnesia, [queue_vhost_field]}).
 -rabbit_upgrade({exchange_options,      mnesia, [operator_policies]}).
--rabbit_upgrade({node_maintenance_states, mnesia, []}).
 
 %% -------------------------------------------------------------------
 
@@ -635,13 +634,6 @@ exchange_options(Table) ->
       [name, type, durable, auto_delete, internal, arguments, scratches, policy,
        operator_policy, decorators, options]).
 
--spec node_maintenance_states() -> 'ok'.
-node_maintenance_states() ->
-    create(rabbit_node_maintenance_states,
-        [{record_name, node_maintenance_state},
-         {attributes, [node, state, context]},
-         {disc_copies, [node()]}]).
-
 %%--------------------------------------------------------------------
 
 transform(TableName, Fun, FieldList) ->