Merge pull request #1 from rabbitmq/master

Merge from source

3 files changed, 52 insertions, 215 deletions

diff --git a/test/health_check_SUITE.erl b/test/health_check_SUITE.erl
deleted file mode 100644
index 5bb55a9289..0000000000
--- a/test/health_check_SUITE.erl
+++ /dev/null
@@ -1,190 +0,0 @@
-%% This Source Code Form is subject to the terms of the Mozilla Public
-%% License, v. 2.0. If a copy of the MPL was not distributed with this
-%% file, You can obtain one at https://mozilla.org/MPL/2.0/.
-%%
-%% Copyright (c) 2016-2020 VMware, Inc. or its affiliates.  All rights reserved.
-%%
-
--module(health_check_SUITE).
-
--include_lib("common_test/include/ct.hrl").
--include_lib("amqp_client/include/amqp_client.hrl").
-
--export([all/0
-        ,groups/0
-        ,init_per_suite/1
-        ,end_per_suite/1
-        ,init_per_group/2
-        ,end_per_group/2
-        ,init_per_testcase/2
-        ,end_per_testcase/2
-        ]).
-
--export([ignores_remote_dead_channel/1
-        ,detects_local_dead_channel/1
-        ,ignores_remote_dead_queue/1
-        ,detects_local_dead_queue/1
-        ,ignores_remote_alarms/1
-        ,detects_local_alarm/1
-        ,honors_timeout_argument/1
-        ,detects_stuck_local_node_monitor/1
-        ,ignores_stuck_remote_node_monitor/1
-        ]).
-
-all() ->
-    [{group, all_cases}].
-
-groups() ->
-    [{all_cases, [],
-      [ignores_remote_dead_queue
-      ,detects_local_dead_queue
-      ,ignores_remote_dead_channel
-      ,detects_local_dead_channel
-      ,ignores_remote_alarms
-      ,detects_local_alarm
-      ,honors_timeout_argument
-      ,detects_stuck_local_node_monitor
-      ,ignores_stuck_remote_node_monitor
-      ]}].
-
-%% -------------------------------------------------------------------
-%% Testsuite setup/teardown.
-%% -------------------------------------------------------------------
-
-init_per_suite(Config) ->
-    rabbit_ct_helpers:log_environment(),
-    rabbit_ct_helpers:run_setup_steps(Config).
-
-end_per_suite(Config) ->
-    rabbit_ct_helpers:run_teardown_steps(Config).
-
-init_per_group(_, Config) ->
-    Config.
-
-end_per_group(_, Config) ->
-    Config.
-
-init_per_testcase(Testcase, Config) ->
-    rabbit_ct_helpers:testcase_started(Config, Testcase),
-    ClusterSize = 2,
-    TestNumber = rabbit_ct_helpers:testcase_number(Config, ?MODULE, Testcase),
-    Config1 = rabbit_ct_helpers:set_config(Config, [
-        {rmq_nodes_count, ClusterSize},
-        {rmq_nodename_suffix, Testcase},
-        {tcp_ports_base, {skip_n_nodes, TestNumber * ClusterSize}}
-      ]),
-    rabbit_ct_helpers:run_steps(Config1,
-      rabbit_ct_broker_helpers:setup_steps() ++
-      rabbit_ct_client_helpers:setup_steps()).
-
-end_per_testcase(Testcase, Config) ->
-    Config1 = rabbit_ct_helpers:run_steps(Config,
-      rabbit_ct_client_helpers:teardown_steps() ++
-      rabbit_ct_broker_helpers:teardown_steps()),
-    rabbit_ct_helpers:testcase_finished(Config1, Testcase).
-
-%%----------------------------------------------------------------------------
-%% Test cases
-%%----------------------------------------------------------------------------
-ignores_remote_dead_channel(Config) ->
-    [A, B] = open_channel_and_declare_queue_everywhere(Config),
-    CPid = suspend_single_channel(Config, B),
-    {ok, _} = rabbit_ct_broker_helpers:rabbitmqctl(Config, A, ["-t", "5", "node_health_check"]),
-    resume_sys_process(Config, B, CPid),
-    ok.
-
-detects_local_dead_channel(Config) ->
-    [A|_] = open_channel_and_declare_queue_everywhere(Config),
-    CPid = suspend_single_channel(Config, A),
-    {error, 75, Str} = rabbit_ct_broker_helpers:rabbitmqctl(Config, A, ["-t", "5", "node_health_check"]),
-    {match, _} = re:run(Str, "operation node_health_check.*timed out"),
-    resume_sys_process(Config, A, CPid),
-    ok.
-
-ignores_remote_dead_queue(Config) ->
-    [A, B] = open_channel_and_declare_queue_everywhere(Config),
-    QPid = suspend_single_queue(Config, B),
-    {ok, _} = rabbit_ct_broker_helpers:rabbitmqctl(Config, A, ["-t", "5", "node_health_check"]),
-    resume_sys_process(Config, B, QPid),
-    ok.
-
-detects_local_dead_queue(Config) ->
-    [A|_] = open_channel_and_declare_queue_everywhere(Config),
-    QPid = suspend_single_queue(Config, A),
-    {error, 75, Str} = rabbit_ct_broker_helpers:rabbitmqctl(Config, A, ["-t", "5", "node_health_check"]),
-    {match, _} = re:run(Str, "operation node_health_check.*timed out"),
-    resume_sys_process(Config, A, QPid),
-    ok.
-
-ignores_remote_alarms(Config) ->
-    [A, B] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename),
-    rabbit_ct_broker_helpers:rabbitmqctl(Config, B,
-                                         ["set_vm_memory_high_watermark", "0.000000001"]),
-    {ok, _} = rabbit_ct_broker_helpers:rabbitmqctl(Config, A, ["-t", "5", "node_health_check"]),
-    ok.
-
-detects_local_alarm(Config) ->
-    [A|_] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename),
-    rabbit_ct_broker_helpers:rabbitmqctl(Config, A,
-                                         ["set_vm_memory_high_watermark", "0.000000001"]),
-    {error, 70, Str} = rabbit_ct_broker_helpers:rabbitmqctl(Config, A, ["-t", "5", "node_health_check"]),
-    {match, _} = re:run(Str, "resource alarm.*in effect"),
-    ok.
-
-detects_stuck_local_node_monitor(Config) ->
-    [A|_] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename),
-    rabbit_ct_broker_helpers:rpc(Config, A, sys, suspend, [rabbit_node_monitor]),
-    {error, 75, Str} = rabbit_ct_broker_helpers:rabbitmqctl(Config, A, ["-t", "5", "node_health_check"]),
-    {match, _} = re:run(Str, "operation node_health_check.*timed out"),
-    resume_sys_process(Config, A, rabbit_node_monitor),
-    ok.
-
-ignores_stuck_remote_node_monitor(Config) ->
-    [A, B] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename),
-    rabbit_ct_broker_helpers:rpc(Config, A, sys, suspend, [rabbit_node_monitor]),
-    {ok, _} = rabbit_ct_broker_helpers:rabbitmqctl(Config, B, ["-t", "5", "node_health_check"]),
-    resume_sys_process(Config, A, rabbit_node_monitor),
-    ok.
-
-honors_timeout_argument(Config) ->
-    [A|_] = open_channel_and_declare_queue_everywhere(Config),
-    QPid = suspend_single_queue(Config, A),
-
-    case timer:tc(rabbit_ct_broker_helpers, rabbitmqctl, [Config, A, ["-t", "5", "node_health_check"]]) of
-        {TimeSpent, {error, 75, _}} ->
-            if TimeSpent < 5000000  -> exit({too_fast, TimeSpent});
-               TimeSpent > 10000000 -> exit({too_slow, TimeSpent}); %% +5 seconds for rabbitmqctl overhead
-               true -> ok
-            end;
-        {_, Unexpected} ->
-            exit({unexpected, Unexpected})
-    end,
-    resume_sys_process(Config, A, QPid),
-    ok.
-
-%%----------------------------------------------------------------------------
-%% Helpers
-%%----------------------------------------------------------------------------
-open_channel_and_declare_queue_everywhere(Config) ->
-    Nodes = rabbit_ct_broker_helpers:get_node_configs(Config, nodename),
-    lists:foreach(fun(Node) ->
-                      Ch = rabbit_ct_client_helpers:open_channel(Config, Node),
-                      #'queue.declare_ok'{} = amqp_channel:call(Ch, #'queue.declare'{})
-                  end,
-                  Nodes),
-    Nodes.
-
-suspend_single_queue(Config, Node) ->
-    [QPid|_] = [rabbit_amqqueue:pid_of(Q) || Q <- rabbit_ct_broker_helpers:rpc(Config, Node, rabbit_amqqueue, list, []),
-                                             Node == node(rabbit_amqqueue:pid_of(Q))],
-    rabbit_ct_broker_helpers:rpc(Config, Node, sys, suspend, [QPid]),
-    QPid.
-
-suspend_single_channel(Config, Node) ->
-    [CPid|_] = [Pid || Pid <- rabbit_ct_broker_helpers:rpc(Config, Node, rabbit_channel, list_local, []),
-                       Node == node(Pid)],
-    rabbit_ct_broker_helpers:rpc(Config, Node, sys, suspend, [CPid]),
-    CPid.
-
-resume_sys_process(Config, Node, Pid) ->
-    rabbit_ct_broker_helpers:rpc(Config, Node, sys, resume, [Pid]).
diff --git a/test/maintenance_mode_SUITE.erl b/test/maintenance_mode_SUITE.erl
index 6ec3f61ad5..61a89c986e 100644
--- a/test/maintenance_mode_SUITE.erl
+++ b/test/maintenance_mode_SUITE.erl
@@ -57,11 +57,21 @@ init_per_testcase(Testcase, Config) ->
         {rmq_nodename_suffix, Testcase},
         {tcp_ports_base, {skip_n_nodes, TestNumber * ClusterSize}}
       ]),
-    rabbit_ct_helpers:run_steps(Config1,
-      rabbit_ct_broker_helpers:setup_steps() ++
-      rabbit_ct_client_helpers:setup_steps() ++ [
-        fun rabbit_ct_broker_helpers:set_ha_policy_all/1
-      ]).
+    Config2 = rabbit_ct_helpers:run_steps(
+                Config1,
+                rabbit_ct_broker_helpers:setup_steps() ++
+                rabbit_ct_client_helpers:setup_steps() ++
+                [fun rabbit_ct_broker_helpers:set_ha_policy_all/1]),
+    FFEnabled = rabbit_ct_broker_helpers:enable_feature_flag(
+                  Config2,
+                  maintenance_mode_status),
+    case FFEnabled of
+        ok ->
+            Config2;
+        Skip ->
+            end_per_testcase(Testcase, Config2),
+            Skip
+    end.
 
 end_per_testcase(Testcase, Config) ->
     Config1 = rabbit_ct_helpers:run_steps(Config,
@@ -75,12 +85,12 @@ end_per_testcase(Testcase, Config) ->
 
 maintenance_mode_status(Config) ->
     Nodes = [A, B, C] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename),
-    
+
     [begin
          ?assertNot(rabbit_ct_broker_helpers:is_being_drained_local_read(Config, Node)),
          ?assertNot(rabbit_ct_broker_helpers:is_being_drained_consistent_read(Config, Node))
      end || Node <- Nodes],
-    
+
     [begin
         [begin
              ?assertNot(rabbit_ct_broker_helpers:is_being_drained_consistent_read(Config, TargetNode, NodeToCheck))
@@ -91,38 +101,38 @@ maintenance_mode_status(Config) ->
     rabbit_ct_helpers:await_condition(
         fun () -> rabbit_ct_broker_helpers:is_being_drained_local_read(Config, B) end,
         10000),
-    
+
     [begin
          ?assert(rabbit_ct_broker_helpers:is_being_drained_consistent_read(Config, TargetNode, B))
      end || TargetNode <- Nodes],
-    
+
     ?assertEqual(
         lists:usort([A, C]),
         lists:usort(rabbit_ct_broker_helpers:rpc(Config, B,
                         rabbit_maintenance, primary_replica_transfer_candidate_nodes, []))),
-    
+
     rabbit_ct_broker_helpers:unmark_as_being_drained(Config, B),
     rabbit_ct_helpers:await_condition(
         fun () -> not rabbit_ct_broker_helpers:is_being_drained_local_read(Config, B) end,
         10000),
-    
+
     [begin
          ?assertNot(rabbit_ct_broker_helpers:is_being_drained_local_read(Config, TargetNode, B)),
          ?assertNot(rabbit_ct_broker_helpers:is_being_drained_consistent_read(Config, TargetNode, B))
      end || TargetNode <- Nodes],
-    
+
     ?assertEqual(
         lists:usort([A, C]),
         lists:usort(rabbit_ct_broker_helpers:rpc(Config, B,
                         rabbit_maintenance, primary_replica_transfer_candidate_nodes, []))),
-    
+
     ok.
 
 
 listener_suspension_status(Config) ->
     Nodes = [A | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename),
     ct:pal("Picked node ~s for maintenance tests...", [A]),
-    
+
     rabbit_ct_helpers:await_condition(
         fun () -> not rabbit_ct_broker_helpers:is_being_drained_local_read(Config, A) end, 10000),
 
@@ -208,14 +218,14 @@ quorum_queue_leadership_transfer(Config) ->
     amqp_channel:call(Ch, #'queue.declare'{queue = QName, durable = true, arguments = [
         {<<"x-queue-type">>, longstr, <<"quorum">>}
     ]}),
-    
+
     %% we cannot assert on the number of local leaders here: declaring a QQ on node A
     %% does not guarantee that the leader will be hosted on node A
-    
+
     rabbit_ct_broker_helpers:drain_node(Config, A),
     rabbit_ct_helpers:await_condition(
         fun () -> rabbit_ct_broker_helpers:is_being_drained_local_read(Config, A) end, 10000),
-    
+
     %% quorum queue leader election is asynchronous
     rabbit_ct_helpers:await_condition(
         fun () ->
diff --git a/test/queue_master_location_SUITE.erl b/test/queue_master_location_SUITE.erl
index 6a346c5618..fab3eac3f0 100644
--- a/test/queue_master_location_SUITE.erl
+++ b/test/queue_master_location_SUITE.erl
@@ -53,11 +53,11 @@ groups() ->
           calculate_random,
           calculate_client_local
         ]},
-        
+
       {maintenance_mode, [], [
           declare_with_min_masters_and_some_nodes_under_maintenance,
           declare_with_min_masters_and_all_nodes_under_maintenance,
-          
+
           declare_with_random_and_some_nodes_under_maintenance,
           declare_with_random_and_all_nodes_under_maintenance
       ]}
@@ -101,9 +101,26 @@ init_per_testcase(Testcase, Config) ->
         {rmq_nodename_suffix, Testcase},
         {tcp_ports_base, {skip_n_nodes, TestNumber * ClusterSize}}
       ]),
-    rabbit_ct_helpers:run_steps(Config1,
-      rabbit_ct_broker_helpers:setup_steps() ++
-      rabbit_ct_client_helpers:setup_steps()).
+    Config2 = rabbit_ct_helpers:run_steps(
+                Config1,
+                rabbit_ct_broker_helpers:setup_steps() ++
+                rabbit_ct_client_helpers:setup_steps()),
+    Group = proplists:get_value(name, ?config(tc_group_properties, Config)),
+    FFEnabled = case Group of
+                    maintenance_mode ->
+                        rabbit_ct_broker_helpers:enable_feature_flag(
+                          Config2,
+                          maintenance_mode_status);
+                    _ ->
+                        ok
+                end,
+    case FFEnabled of
+        ok ->
+            Config2;
+        Skip ->
+            end_per_testcase(Testcase, Config2),
+            Skip
+    end.
 
 end_per_testcase(Testcase, Config) ->
     Config1 = rabbit_ct_helpers:run_steps(Config,
@@ -226,7 +243,7 @@ declare_with_min_masters_and_some_nodes_under_maintenance(Config) ->
     set_location_policy(Config, ?POLICY, <<"min-masters">>),
     rabbit_ct_broker_helpers:mark_as_being_drained(Config, 0),
     rabbit_ct_broker_helpers:mark_as_being_drained(Config, 1),
-    
+
     QName = <<"qm.tests.min_masters.maintenance.case1">>,
     Resource = rabbit_misc:r(<<"/">>, queue, QName),
     Record = declare(Config, Resource, false, false, _Args = [], none),
@@ -244,7 +261,7 @@ declare_with_random_and_some_nodes_under_maintenance(Config) ->
     set_location_policy(Config, ?POLICY, <<"random">>),
     rabbit_ct_broker_helpers:mark_as_being_drained(Config, 0),
     rabbit_ct_broker_helpers:mark_as_being_drained(Config, 2),
-    
+
     QName = <<"qm.tests.random.maintenance.case1">>,
     Resource = rabbit_misc:r(<<"/">>, queue, QName),
     Record = declare(Config, Resource, false, false, _Args = [], none),
@@ -263,7 +280,7 @@ declare_with_all_nodes_under_maintenance(Config, Locator) ->
     rabbit_ct_broker_helpers:mark_as_being_drained(Config, 0),
     rabbit_ct_broker_helpers:mark_as_being_drained(Config, 1),
     rabbit_ct_broker_helpers:mark_as_being_drained(Config, 2),
-    
+
     QName = rabbit_data_coercion:to_binary(
         rabbit_misc:format("qm.tests.~s.maintenance.case2", [Locator])),
     Resource = rabbit_misc:r(<<"/">>, queue, QName),