Add per_user_connection_channel_limit_partitions test

1 files changed, 183 insertions, 0 deletions

diff --git a/test/per_user_connection_channel_limit_partitions_SUITE.erl b/test/per_user_connection_channel_limit_partitions_SUITE.erl
new file mode 100644
index 0000000000..d737a4f51e
--- /dev/null
+++ b/test/per_user_connection_channel_limit_partitions_SUITE.erl
@@ -0,0 +1,183 @@
+%% The contents of this file are subject to the Mozilla Public License
+%% Version 1.1 (the "License"); you may not use this file except in
+%% compliance with the License. You may obtain a copy of the License at
+%% https://www.mozilla.org/MPL/
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the
+%% License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% The Original Code is RabbitMQ.
+%%
+%% The Initial Developer of the Original Code is GoPivotal, Inc.
+%% Copyright (c) 2011-2020 VMware, Inc. or its affiliates.  All rights reserved.
+%%
+
+-module(per_user_connection_channel_limit_partitions_SUITE).
+
+-include_lib("common_test/include/ct.hrl").
+-include_lib("amqp_client/include/amqp_client.hrl").
+-include_lib("eunit/include/eunit.hrl").
+
+-compile(export_all).
+
+-import(rabbit_ct_client_helpers, [open_unmanaged_connection/2,
+                                   open_unmanaged_connection/3]).
+
+all() ->
+    [
+     {group, net_ticktime_1}
+    ].
+
+groups() ->
+    [
+     {net_ticktime_1, [], [
+          cluster_full_partition_with_autoheal
+     ]}
+    ].
+
+suite() ->
+    [
+      %% If a test hangs, no need to wait for 30 minutes.
+      {timetrap, {minutes, 8}}
+    ].
+
+%% see partitions_SUITE
+-define(DELAY, 12000).
+
+%% -------------------------------------------------------------------
+%% Testsuite setup/teardown.
+%% -------------------------------------------------------------------
+
+init_per_suite(Config) ->
+    rabbit_ct_helpers:log_environment(),
+    rabbit_ct_helpers:run_setup_steps(Config, [
+      fun rabbit_ct_broker_helpers:configure_dist_proxy/1
+    ]).
+
+end_per_suite(Config) ->
+    rabbit_ct_helpers:run_teardown_steps(Config).
+
+init_per_group(net_ticktime_1 = Group, Config) ->
+    Config1 = rabbit_ct_helpers:set_config(Config, [{net_ticktime, 1}]),
+    init_per_multinode_group(Group, Config1, 3).
+
+init_per_multinode_group(_Group, Config, NodeCount) ->
+    Suffix = rabbit_ct_helpers:testcase_absname(Config, "", "-"),
+    Config1 = rabbit_ct_helpers:set_config(Config, [
+                                                    {rmq_nodes_count, NodeCount},
+                                                    {rmq_nodename_suffix, Suffix}
+      ]),
+    rabbit_ct_helpers:run_steps(Config1,
+      rabbit_ct_broker_helpers:setup_steps() ++
+      rabbit_ct_client_helpers:setup_steps()).
+
+end_per_group(_Group, Config) ->
+    rabbit_ct_helpers:run_steps(Config,
+      rabbit_ct_client_helpers:teardown_steps() ++
+      rabbit_ct_broker_helpers:teardown_steps()).
+
+init_per_testcase(Testcase, Config) ->
+    rabbit_ct_helpers:testcase_started(Config, Testcase).
+
+end_per_testcase(Testcase, Config) ->
+    rabbit_ct_helpers:testcase_finished(Config, Testcase).
+
+%% -------------------------------------------------------------------
+%% Test cases.
+%% -------------------------------------------------------------------
+
+cluster_full_partition_with_autoheal(Config) ->
+    Username = proplists:get_value(rmq_username, Config),
+    rabbit_ct_broker_helpers:set_partition_handling_mode_globally(Config, autoheal),
+
+    ?assertEqual(0, count_connections_in(Config, Username)),
+    [A, B, C] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename),
+
+    %% 6 connections, 2 per node
+    Conn1 = open_unmanaged_connection(Config, A),
+    Conn2 = open_unmanaged_connection(Config, A),
+    Conn3 = open_unmanaged_connection(Config, B),
+    Conn4 = open_unmanaged_connection(Config, B),
+    Conn5 = open_unmanaged_connection(Config, C),
+    Conn6 = open_unmanaged_connection(Config, C),
+
+    _Chans1 = [_|_] = open_channels(Conn1, 5),
+    _Chans3 = [_|_] = open_channels(Conn3, 5),
+    _Chans5 = [_|_] = open_channels(Conn5, 5),
+    wait_for_count_connections_in(Config, Username, 6, 60000),
+    ?assertEqual(15, count_channels_in(Config, Username)),
+
+    %% B drops off the network, non-reachable by either A or C
+    rabbit_ct_broker_helpers:block_traffic_between(A, B),
+    rabbit_ct_broker_helpers:block_traffic_between(B, C),
+    timer:sleep(?DELAY),
+
+    %% A and C are still connected, so 4 connections are tracked
+    %% All connections to B are dropped
+    wait_for_count_connections_in(Config, Username, 4, 60000),
+    ?assertEqual(10, count_channels_in(Config, Username)),
+
+    rabbit_ct_broker_helpers:allow_traffic_between(A, B),
+    rabbit_ct_broker_helpers:allow_traffic_between(B, C),
+    timer:sleep(?DELAY),
+
+    %% during autoheal B's connections were dropped
+    wait_for_count_connections_in(Config, Username, 4, 60000),
+    ?assertEqual(10, count_channels_in(Config, Username)),
+
+    lists:foreach(fun (Conn) ->
+                          (catch rabbit_ct_client_helpers:close_connection(Conn))
+                  end, [Conn1, Conn2, Conn3, Conn4,
+                        Conn5, Conn6]),
+    ?assertEqual(0, count_connections_in(Config, Username)),
+    ?assertEqual(0, count_channels_in(Config, Username)),
+
+    passed.
+
+%% -------------------------------------------------------------------
+%% Helpers
+%% -------------------------------------------------------------------
+
+wait_for_count_connections_in(Config, Username, Expected, Time) when Time =< 0 ->
+    ?assertMatch(Connections when length(Connections) == Expected,
+                                  connections_in(Config, Username));
+wait_for_count_connections_in(Config, Username, Expected, Time) ->
+    case connections_in(Config, Username) of
+        Connections when length(Connections) == Expected ->
+            ok;
+        _ ->
+            Sleep = 3000,
+            timer:sleep(Sleep),
+            wait_for_count_connections_in(Config, Username, Expected, Time - Sleep)
+    end.
+
+open_channels(Conn, N) ->
+    [begin
+        {ok, Ch} = amqp_connection:open_channel(Conn),
+        Ch
+     end || _ <- lists:seq(1, N)].
+
+count_connections_in(Config, Username) ->
+    length(connections_in(Config, Username)).
+
+connections_in(Config, Username) ->
+    connections_in(Config, 0, Username).
+connections_in(Config, NodeIndex, Username) ->
+    tracked_list_of_user(Config, NodeIndex, rabbit_connection_tracking, Username).
+
+count_channels_in(Config, Username) ->
+    Channels = channels_in(Config, Username),
+    length([Ch || Ch = #tracked_channel{username = Username0} <- Channels,
+                  Username =:= Username0]).
+
+channels_in(Config, Username) ->
+    channels_in(Config, 0, Username).
+channels_in(Config, NodeIndex, Username) ->
+    tracked_list_of_user(Config, NodeIndex, rabbit_channel_tracking, Username).
+
+tracked_list_of_user(Config, NodeIndex, TrackingMod, Username) ->
+    rabbit_ct_broker_helpers:rpc(Config, NodeIndex,
+                                 TrackingMod,
+                                 list_of_user, [Username]).