diff options
| author | Michael Klishin <michael@clojurewerkz.org> | 2020-08-31 08:37:22 +0300 |
|---|---|---|
| committer | Michael Klishin <michael@clojurewerkz.org> | 2020-08-31 08:37:22 +0300 |
| commit | 58a3b450ab3cabe29576bfb6504449c94ae1ac62 (patch) | |
| tree | 8e59bacdd4b2bcafaf12f2e3444e5e7a314d3be5 /test | |
| parent | 716d293e0dfc727700dfb93be64081c5cbf6dd5a (diff) | |
| parent | 37641110273bdfe274d378a549cb9343980be8f2 (diff) | |
| download | rabbitmq-server-git-58a3b450ab3cabe29576bfb6504449c94ae1ac62.tar.gz | |
Merge branch 'Ayanda-D-rabbitmq-per-user-connection-channel-limits'
Diffstat (limited to 'test')
| -rw-r--r-- | test/per_user_connection_channel_limit_SUITE.erl | 1625 | ||||
| -rw-r--r-- | test/per_user_connection_channel_limit_partitions_SUITE.erl | 174 | ||||
| -rw-r--r-- | test/per_user_connection_channel_tracking_SUITE.erl | 840 | ||||
| -rw-r--r-- | test/per_vhost_connection_limit_SUITE.erl | 4 | ||||
| -rw-r--r-- | test/per_vhost_connection_limit_partitions_SUITE.erl | 21 | ||||
| -rw-r--r-- | test/unit_access_control_SUITE.erl | 29 | ||||
| -rw-r--r-- | test/vhost_SUITE.erl | 2 |
7 files changed, 2653 insertions, 42 deletions
diff --git a/test/per_user_connection_channel_limit_SUITE.erl b/test/per_user_connection_channel_limit_SUITE.erl new file mode 100644 index 0000000000..35745d65f8 --- /dev/null +++ b/test/per_user_connection_channel_limit_SUITE.erl @@ -0,0 +1,1625 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(per_user_connection_channel_limit_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-compile(export_all). + +all() -> + [ + {group, cluster_size_1_network}, + {group, cluster_size_2_network}, + {group, cluster_size_2_direct} + ]. + +groups() -> + ClusterSize1Tests = [ + most_basic_single_node_connection_and_channel_count, + single_node_single_user_connection_and_channel_count, + single_node_multiple_users_connection_and_channel_count, + single_node_list_in_user, + single_node_single_user_limit, + single_node_single_user_zero_limit, + single_node_single_user_clear_limits, + single_node_multiple_users_clear_limits, + single_node_multiple_users_limit, + single_node_multiple_users_zero_limit + + ], + ClusterSize2Tests = [ + most_basic_cluster_connection_and_channel_count, + cluster_single_user_connection_and_channel_count, + cluster_multiple_users_connection_and_channel_count, + cluster_node_restart_connection_and_channel_count, + cluster_node_list_on_node, + cluster_single_user_limit, + cluster_single_user_limit2, + cluster_single_user_zero_limit, + cluster_single_user_clear_limits, + cluster_multiple_users_clear_limits, + cluster_multiple_users_zero_limit + ], + [ + {cluster_size_1_network, [], ClusterSize1Tests}, + {cluster_size_2_network, [], ClusterSize2Tests}, + {cluster_size_2_direct, [], ClusterSize2Tests} + ]. + +suite() -> + [ + %% If a test hangs, no need to wait for 30 minutes. + {timetrap, {minutes, 8}} + ]. + +%% ------------------------------------------------------------------- +%% Testsuite setup/teardown. +%% ------------------------------------------------------------------- + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(cluster_size_1_network, Config) -> + Config1 = rabbit_ct_helpers:set_config(Config, [{connection_type, network}]), + init_per_multinode_group(cluster_size_1_network, Config1, 1); +init_per_group(cluster_size_2_network, Config) -> + Config1 = rabbit_ct_helpers:set_config(Config, [{connection_type, network}]), + init_per_multinode_group(cluster_size_2_network, Config1, 2); +init_per_group(cluster_size_2_direct, Config) -> + Config1 = rabbit_ct_helpers:set_config(Config, [{connection_type, direct}]), + init_per_multinode_group(cluster_size_2_direct, Config1, 2); + +init_per_group(cluster_rename, Config) -> + init_per_multinode_group(cluster_rename, Config, 2). + +init_per_multinode_group(Group, Config, NodeCount) -> + Suffix = rabbit_ct_helpers:testcase_absname(Config, "", "-"), + Config1 = rabbit_ct_helpers:set_config(Config, [ + {rmq_nodes_count, NodeCount}, + {rmq_nodename_suffix, Suffix} + ]), + case Group of + cluster_rename -> + % The broker is managed by {init,end}_per_testcase(). + Config1; + _ -> + rabbit_ct_helpers:run_steps(Config1, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()) + end. + +end_per_group(cluster_rename, Config) -> + % The broker is managed by {init,end}_per_testcase(). + Config; +end_per_group(_Group, Config) -> + rabbit_ct_helpers:run_steps(Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()). + +init_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_started(Config, Testcase), + clear_all_connection_tracking_tables(Config), + clear_all_channel_tracking_tables(Config), + Config. + +end_per_testcase(Testcase, Config) -> + clear_all_connection_tracking_tables(Config), + clear_all_channel_tracking_tables(Config), + rabbit_ct_helpers:testcase_finished(Config, Testcase). + +clear_all_connection_tracking_tables(Config) -> + [rabbit_ct_broker_helpers:rpc(Config, + N, + rabbit_connection_tracking, + clear_tracking_tables, + []) || N <- rabbit_ct_broker_helpers:get_node_configs(Config, nodename)]. + +clear_all_channel_tracking_tables(Config) -> + [rabbit_ct_broker_helpers:rpc(Config, + N, + rabbit_channel_tracking, + clear_tracking_tables, + []) || N <- rabbit_ct_broker_helpers:get_node_configs(Config, nodename)]. + +%% ------------------------------------------------------------------- +%% Test cases. +%% ------------------------------------------------------------------- + +most_basic_single_node_connection_and_channel_count(Config) -> + Username = proplists:get_value(rmq_username, Config), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 0 andalso + count_channels_of_user(Config, Username) =:= 0 + end), + + [Conn] = open_connections(Config, [0]), + [Chan] = open_channels(Conn, 1), + + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 1 andalso + count_channels_of_user(Config, Username) =:= 1 + end), + close_channels([Chan]), + rabbit_ct_helpers:await_condition( + fun () -> + count_channels_of_user(Config, Username) =:= 0 + end), + close_connections([Conn]), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 0 + end). + +single_node_single_user_connection_and_channel_count(Config) -> + Username = proplists:get_value(rmq_username, Config), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 0 andalso + count_channels_of_user(Config, Username) =:= 0 + end), + + [Conn1] = open_connections(Config, [0]), + [Chan1] = open_channels(Conn1, 1), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 1 andalso + count_channels_of_user(Config, Username) =:= 1 + end), + close_channels([Chan1]), + rabbit_ct_helpers:await_condition( + fun () -> + count_channels_of_user(Config, Username) =:= 0 + end), + close_connections([Conn1]), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 0 + end), + + [Conn2] = open_connections(Config, [0]), + Chans2 = [_|_] = open_channels(Conn2, 5), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 1 andalso + count_channels_of_user(Config, Username) =:= 5 + end), + + [Conn3] = open_connections(Config, [0]), + Chans3 = [_|_] = open_channels(Conn3, 5), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 2 andalso + count_channels_of_user(Config, Username) =:= 10 + end), + + [Conn4] = open_connections(Config, [0]), + _Chans4 = [_|_] = open_channels(Conn4, 5), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 3 andalso + count_channels_of_user(Config, Username) =:= 15 + end), + + close_connections([Conn4]), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 2 andalso + count_channels_of_user(Config, Username) =:= 10 + end), + + [Conn5] = open_connections(Config, [0]), + Chans5 = [_|_] = open_channels(Conn5, 5), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 3 andalso + count_channels_of_user(Config, Username) =:= 15 + end), + + close_channels(Chans2 ++ Chans3 ++ Chans5), + rabbit_ct_helpers:await_condition( + fun () -> + count_channels_of_user(Config, Username) =:= 0 + end), + + close_connections([Conn2, Conn3, Conn5]), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 0 + end). + +single_node_multiple_users_connection_and_channel_count(Config) -> + Username1 = <<"guest1">>, + Username2 = <<"guest2">>, + + set_up_user(Config, Username1), + set_up_user(Config, Username2), + + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username1) =:= 0 andalso + count_channels_of_user(Config, Username1) =:= 0 + end), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username2) =:= 0 andalso + count_channels_of_user(Config, Username2) =:= 0 + end), + + [Conn1] = open_connections(Config, [{0, Username1}]), + Chans1 = [_|_] = open_channels(Conn1, 5), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username1) =:= 1 andalso + count_channels_of_user(Config, Username1) =:= 5 + end), + close_channels(Chans1), + rabbit_ct_helpers:await_condition( + fun () -> + count_channels_of_user(Config, Username1) =:= 0 + end), + ?assertEqual(0, count_channels_of_user(Config, Username1)), + close_connections([Conn1]), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username1) =:= 0 andalso + count_channels_of_user(Config, Username1) =:= 0 + end), + + [Conn2] = open_connections(Config, [{0, Username2}]), + Chans2 = [_|_] = open_channels(Conn2, 5), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username2) =:= 1 andalso + count_channels_of_user(Config, Username2) =:= 5 + end), + + [Conn3] = open_connections(Config, [{0, Username1}]), + Chans3 = [_|_] = open_channels(Conn3, 5), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username1) =:= 1 andalso + count_channels_of_user(Config, Username1) =:= 5 + end), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username2) =:= 1 andalso + count_channels_of_user(Config, Username2) =:= 5 + end), + + [Conn4] = open_connections(Config, [{0, Username1}]), + _Chans4 = [_|_] = open_channels(Conn4, 5), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username1) =:= 2 andalso + count_channels_of_user(Config, Username1) =:= 10 + end), + + close_connections([Conn4]), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username1) =:= 1 andalso + count_channels_of_user(Config, Username1) =:= 5 + end), + + [Conn5] = open_connections(Config, [{0, Username2}]), + Chans5 = [_|_] = open_channels(Conn5, 5), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username2) =:= 2 andalso + count_channels_of_user(Config, Username2) =:= 10 + end), + + [Conn6] = open_connections(Config, [{0, Username2}]), + Chans6 = [_|_] = open_channels(Conn6, 5), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username2) =:= 3 andalso + count_channels_of_user(Config, Username2) =:= 15 + end), + + close_channels(Chans2 ++ Chans3 ++ Chans5 ++ Chans6), + rabbit_ct_helpers:await_condition( + fun () -> + count_channels_of_user(Config, Username1) =:= 0 andalso + count_channels_of_user(Config, Username2) =:= 0 + end), + + close_connections([Conn2, Conn3, Conn5, Conn6]), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username1) =:= 0 andalso + count_connections_of_user(Config, Username2) =:= 0 + end), + + rabbit_ct_broker_helpers:delete_user(Config, Username1), + rabbit_ct_broker_helpers:delete_user(Config, Username2). + +single_node_list_in_user(Config) -> + Username1 = <<"guest1">>, + Username2 = <<"guest2">>, + + set_up_user(Config, Username1), + set_up_user(Config, Username2), + + ?assertEqual(0, length(connections_in(Config, Username1))), + ?assertEqual(0, length(connections_in(Config, Username2))), + ?assertEqual(0, length(channels_in(Config, Username1))), + ?assertEqual(0, length(channels_in(Config, Username2))), + + [Conn1] = open_connections(Config, [{0, Username1}]), + [Chan1] = open_channels(Conn1, 1), + [#tracked_connection{username = Username1}] = connections_in(Config, Username1), + [#tracked_channel{username = Username1}] = channels_in(Config, Username1), + close_channels([Chan1]), + ?assertEqual(0, length(channels_in(Config, Username1))), + close_connections([Conn1]), + ?assertEqual(0, length(connections_in(Config, Username1))), + + [Conn2] = open_connections(Config, [{0, Username2}]), + [Chan2] = open_channels(Conn2, 1), + [#tracked_connection{username = Username2}] = connections_in(Config, Username2), + [#tracked_channel{username = Username2}] = channels_in(Config, Username2), + + [Conn3] = open_connections(Config, [{0, Username1}]), + [Chan3] = open_channels(Conn3, 1), + [#tracked_connection{username = Username1}] = connections_in(Config, Username1), + [#tracked_channel{username = Username1}] = channels_in(Config, Username1), + + [Conn4] = open_connections(Config, [{0, Username1}]), + [_Chan4] = open_channels(Conn4, 1), + close_connections([Conn4]), + [#tracked_connection{username = Username1}] = connections_in(Config, Username1), + [#tracked_channel{username = Username1}] = channels_in(Config, Username1), + + [Conn5, Conn6] = open_connections(Config, [{0, Username2}, {0, Username2}]), + [Chan5] = open_channels(Conn5, 1), + [Chan6] = open_channels(Conn6, 1), + [<<"guest1">>, <<"guest2">>] = + lists:usort(lists:map(fun (#tracked_connection{username = V}) -> V end, + all_connections(Config))), + [<<"guest1">>, <<"guest2">>] = + lists:usort(lists:map(fun (#tracked_channel{username = V}) -> V end, + all_channels(Config))), + + close_channels([Chan2, Chan3, Chan5, Chan6]), + ?assertEqual(0, length(all_channels(Config))), + + close_connections([Conn2, Conn3, Conn5, Conn6]), + ?assertEqual(0, length(all_connections(Config))), + + rabbit_ct_broker_helpers:delete_user(Config, Username1), + rabbit_ct_broker_helpers:delete_user(Config, Username2). + +most_basic_cluster_connection_and_channel_count(Config) -> + Username = proplists:get_value(rmq_username, Config), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 0 andalso + count_channels_of_user(Config, Username) =:= 0 + end), + + [Conn1] = open_connections(Config, [0]), + Chans1 = [_|_] = open_channels(Conn1, 5), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 1 andalso + count_channels_of_user(Config, Username) =:= 5 + end), + ?assertEqual(1, count_connections_of_user(Config, Username)), + ?assertEqual(5, count_channels_of_user(Config, Username)), + + [Conn2] = open_connections(Config, [1]), + Chans2 = [_|_] = open_channels(Conn2, 5), + ?assertEqual(2, count_connections_of_user(Config, Username)), + ?assertEqual(10, count_channels_of_user(Config, Username)), + + [Conn3] = open_connections(Config, [1]), + Chans3 = [_|_] = open_channels(Conn3, 5), + ?assertEqual(3, count_connections_of_user(Config, Username)), + ?assertEqual(15, count_channels_of_user(Config, Username)), + + close_channels(Chans1 ++ Chans2 ++ Chans3), + ?assertEqual(0, count_channels_of_user(Config, Username)), + + close_connections([Conn1, Conn2, Conn3]), + ?assertEqual(0, count_connections_of_user(Config, Username)). + +cluster_single_user_connection_and_channel_count(Config) -> + Username = proplists:get_value(rmq_username, Config), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 0 andalso + count_channels_of_user(Config, Username) =:= 0 + end), + + [Conn1] = open_connections(Config, [0]), + _Chans1 = [_|_] = open_channels(Conn1, 5), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 1 andalso + count_channels_of_user(Config, Username) =:= 5 + end), + + close_connections([Conn1]), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 0 andalso + count_channels_of_user(Config, Username) =:= 0 + end), + + [Conn2] = open_connections(Config, [1]), + Chans2 = [_|_] = open_channels(Conn2, 5), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 1 andalso + count_channels_of_user(Config, Username) =:= 5 + end), + + [Conn3] = open_connections(Config, [0]), + Chans3 = [_|_] = open_channels(Conn3, 5), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 2 andalso + count_channels_of_user(Config, Username) =:= 10 + end), + + [Conn4] = open_connections(Config, [1]), + Chans4 = [_|_] = open_channels(Conn4, 5), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 3 andalso + count_channels_of_user(Config, Username) =:= 15 + end), + + close_channels(Chans2 ++ Chans3 ++ Chans4), + rabbit_ct_helpers:await_condition( + fun () -> + count_channels_of_user(Config, Username) =:= 0 + end), + + close_connections([Conn2, Conn3, Conn4]), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 0 + end). + +cluster_multiple_users_connection_and_channel_count(Config) -> + Username1 = <<"guest1">>, + Username2 = <<"guest2">>, + + set_up_user(Config, Username1), + set_up_user(Config, Username2), + + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username1) =:= 0 andalso + count_connections_of_user(Config, Username2) =:= 0 + end), + rabbit_ct_helpers:await_condition( + fun () -> + count_channels_of_user(Config, Username1) =:= 0 andalso + count_channels_of_user(Config, Username2) =:= 0 + end), + + [Conn1] = open_connections(Config, [{0, Username1}]), + _Chans1 = [_|_] = open_channels(Conn1, 5), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username1) =:= 1 andalso + count_channels_of_user(Config, Username1) =:= 5 + end), + close_connections([Conn1]), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username1) =:= 0 andalso + count_channels_of_user(Config, Username1) =:= 0 + end), + + [Conn2] = open_connections(Config, [{1, Username2}]), + Chans2 = [_|_] = open_channels(Conn2, 5), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username2) =:= 1 andalso + count_channels_of_user(Config, Username2) =:= 5 + end), + + [Conn3] = open_connections(Config, [{1, Username1}]), + Chans3 = [_|_] = open_channels(Conn3, 5), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username1) =:= 1 andalso + count_channels_of_user(Config, Username1) =:= 5 + end), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username2) =:= 1 andalso + count_channels_of_user(Config, Username2) =:= 5 + end), + + [Conn4] = open_connections(Config, [{0, Username1}]), + _Chans4 = [_|_] = open_channels(Conn4, 5), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username1) =:= 2 andalso + count_channels_of_user(Config, Username1) =:= 10 + end), + + close_connections([Conn4]), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username1) =:= 1 andalso + count_channels_of_user(Config, Username1) =:= 5 + end), + + [Conn5] = open_connections(Config, [{1, Username2}]), + Chans5 = [_|_] = open_channels(Conn5, 5), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username2) =:= 2 andalso + count_channels_of_user(Config, Username2) =:= 10 + end), + + [Conn6] = open_connections(Config, [{0, Username2}]), + Chans6 = [_|_] = open_channels(Conn6, 5), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username2) =:= 3 andalso + count_channels_of_user(Config, Username2) =:= 15 + end), + + close_channels(Chans2 ++ Chans3 ++ Chans5 ++ Chans6), + rabbit_ct_helpers:await_condition( + fun () -> + count_channels_of_user(Config, Username1) =:= 0 andalso + count_channels_of_user(Config, Username2) =:= 0 + end), + + close_connections([Conn2, Conn3, Conn5, Conn6]), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username1) =:= 0 andalso + count_connections_of_user(Config, Username2) =:= 0 + end), + + rabbit_ct_broker_helpers:delete_user(Config, Username1), + rabbit_ct_broker_helpers:delete_user(Config, Username2). + +cluster_node_restart_connection_and_channel_count(Config) -> + Username = proplists:get_value(rmq_username, Config), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 0 andalso + count_channels_of_user(Config, Username) =:= 0 + end), + + [Conn1] = open_connections(Config, [0]), + _Chans1 = [_|_] = open_channels(Conn1, 5), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 1 andalso + count_channels_of_user(Config, Username) =:= 5 + end), + close_connections([Conn1]), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 0 andalso + count_channels_of_user(Config, Username) =:= 0 + end), + + [Conn2] = open_connections(Config, [1]), + Chans2 = [_|_] = open_channels(Conn2, 5), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 1 andalso + count_channels_of_user(Config, Username) =:= 5 + end), + + [Conn3] = open_connections(Config, [0]), + Chans3 = [_|_] = open_channels(Conn3, 5), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 2 andalso + count_channels_of_user(Config, Username) =:= 10 + end), + + [Conn4] = open_connections(Config, [1]), + _Chans4 = [_|_] = open_channels(Conn4, 5), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 3 andalso + count_channels_of_user(Config, Username) =:= 15 + end), + + [Conn5] = open_connections(Config, [1]), + Chans5 = [_|_] = open_channels(Conn5, 5), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 4 andalso + count_channels_of_user(Config, Username) =:= 20 + end), + + rabbit_ct_broker_helpers:restart_broker(Config, 1), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 1 andalso + count_channels_of_user(Config, Username) =:= 5 + end), + + close_channels(Chans2 ++ Chans3 ++ Chans5), + rabbit_ct_helpers:await_condition( + fun () -> + count_channels_of_user(Config, Username) =:= 0 + end), + + close_connections([Conn2, Conn3, Conn4, Conn5]), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 0 + end). + +cluster_node_list_on_node(Config) -> + [A, B] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + rabbit_ct_helpers:await_condition( + fun () -> + length(all_connections(Config)) =:= 0 andalso + length(all_channels(Config)) =:= 0 andalso + length(connections_on_node(Config, 0)) =:= 0 andalso + length(channels_on_node(Config, 0)) =:= 0 + end), + + [Conn1] = open_connections(Config, [0]), + _Chans1 = [_|_] = open_channels(Conn1, 5), + [#tracked_connection{node = A}] = connections_on_node(Config, 0), + rabbit_ct_helpers:await_condition( + fun () -> + length([Ch || Ch <- channels_on_node(Config, 0), Ch#tracked_channel.node =:= A]) =:= 5 + end), + close_connections([Conn1]), + rabbit_ct_helpers:await_condition( + fun () -> + length(connections_on_node(Config, 0)) =:= 0 andalso + length(channels_on_node(Config, 0)) =:= 0 + end), + + [Conn2] = open_connections(Config, [1]), + _Chans2 = [_|_] = open_channels(Conn2, 5), + [#tracked_connection{node = B}] = connections_on_node(Config, 1), + rabbit_ct_helpers:await_condition( + fun () -> + length([Ch || Ch <- channels_on_node(Config, 1), Ch#tracked_channel.node =:= B]) =:= 5 + end), + + [Conn3] = open_connections(Config, [0]), + Chans3 = [_|_] = open_channels(Conn3, 5), + rabbit_ct_helpers:await_condition( + fun () -> + length(connections_on_node(Config, 0)) =:= 1 andalso + length(channels_on_node(Config, 0)) =:= 5 + end), + + [Conn4] = open_connections(Config, [1]), + _Chans4 = [_|_] = open_channels(Conn4, 5), + rabbit_ct_helpers:await_condition( + fun () -> + length(connections_on_node(Config, 1)) =:= 2 andalso + length(channels_on_node(Config, 1)) =:= 10 + end), + + close_connections([Conn4]), + rabbit_ct_helpers:await_condition( + fun () -> + length(connections_on_node(Config, 1)) =:= 1 andalso + length(channels_on_node(Config, 1)) =:= 5 + end), + + [Conn5] = open_connections(Config, [0]), + Chans5 = [_|_] = open_channels(Conn5, 5), + rabbit_ct_helpers:await_condition( + fun () -> + length(connections_on_node(Config, 0)) =:= 2 andalso + length(channels_on_node(Config, 0)) =:= 10 + end), + + rabbit_ct_broker_helpers:stop_broker(Config, 1), + await_running_node_refresh(Config, 0), + + rabbit_ct_helpers:await_condition( + fun () -> + length(all_connections(Config)) =:= 2 andalso + length(all_channels(Config)) =:= 10 + end), + + close_channels(Chans3 ++ Chans5), + rabbit_ct_helpers:await_condition( + fun () -> + length(all_channels(Config)) =:= 0 + end), + + close_connections([Conn3, Conn5]), + rabbit_ct_helpers:await_condition( + fun () -> + length(all_connections(Config)) =:= 0 + end), + + rabbit_ct_broker_helpers:start_broker(Config, 1). + +single_node_single_user_limit(Config) -> + single_node_single_user_limit_with(Config, 5, 25), + single_node_single_user_limit_with(Config, -1, -1). + +single_node_single_user_limit_with(Config, ConnLimit, ChLimit) -> + Username = proplists:get_value(rmq_username, Config), + set_user_connection_and_channel_limit(Config, Username, 3, 15), + + ?assertEqual(0, count_connections_of_user(Config, Username)), + ?assertEqual(0, count_channels_of_user(Config, Username)), + + [Conn1, Conn2, Conn3] = Conns1 = open_connections(Config, [0, 0, 0]), + [_Chans1, Chans2, Chans3] = [open_channels(Conn, 5) || Conn <- Conns1], + + %% we've crossed the limit + expect_that_client_connection_is_rejected(Config, 0), + expect_that_client_connection_is_rejected(Config, 0), + expect_that_client_connection_is_rejected(Config, 0), + expect_that_client_channel_is_rejected(Conn1), + + rabbit_ct_helpers:await_condition( + fun () -> + is_process_alive(Conn1) =:= false andalso + is_process_alive(Conn2) andalso + is_process_alive(Conn3) + end), + + set_user_connection_and_channel_limit(Config, Username, ConnLimit, ChLimit), + [Conn4, Conn5] = Conns2 = open_connections(Config, [0, 0]), + [Chans4, Chans5] = [open_channels(Conn, 5) || Conn <- Conns2], + + close_channels(Chans2 ++ Chans3 ++ Chans4 ++ Chans5), + rabbit_ct_helpers:await_condition( + fun () -> + count_channels_of_user(Config, Username) =:= 0 + end), + + close_connections([Conn1, Conn2, Conn3, Conn4, Conn5]), + ?assertEqual(0, count_connections_of_user(Config, Username)), + + set_user_connection_and_channel_limit(Config, Username, -1, -1). + +single_node_single_user_zero_limit(Config) -> + Username = proplists:get_value(rmq_username, Config), + set_user_connection_and_channel_limit(Config, Username, 0, 0), + + ?assertEqual(0, count_connections_of_user(Config, Username)), + ?assertEqual(0, count_channels_of_user(Config, Username)), + + %% with limit = 0 no connections are allowed + expect_that_client_connection_is_rejected(Config), + expect_that_client_connection_is_rejected(Config), + expect_that_client_connection_is_rejected(Config), + + %% with limit = 0 no channels are allowed + set_user_connection_and_channel_limit(Config, Username, 1, 0), + [ConnA] = open_connections(Config, [0]), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 1 + end), + expect_that_client_channel_is_rejected(ConnA), + rabbit_ct_helpers:await_condition( + fun () -> + is_process_alive(ConnA) =:= false andalso + count_connections_of_user(Config, Username) =:= 0 andalso + count_channels_of_user(Config, Username) =:= 0 + end), + + set_user_connection_and_channel_limit(Config, Username, -1, -1), + [Conn1, Conn2] = Conns1 = open_connections(Config, [0, 0]), + [Chans1, Chans2] = [open_channels(Conn, 5) || Conn <- Conns1], + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 2 andalso + count_channels_of_user(Config, Username) =:= 10 + end), + + close_channels(Chans1 ++ Chans2), + rabbit_ct_helpers:await_condition( + fun () -> + count_channels_of_user(Config, Username) =:= 0 + end), + + close_connections([Conn1, Conn2]), + ?assertEqual(0, count_connections_of_user(Config, Username)). + +single_node_single_user_clear_limits(Config) -> + Username = proplists:get_value(rmq_username, Config), + set_user_connection_and_channel_limit(Config, Username, 3, 15), + + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 0 andalso + count_channels_of_user(Config, Username) =:= 0 + end), + + [Conn1, Conn2, Conn3] = Conns1 = open_connections(Config, [0, 0, 0]), + [_Chans1, Chans2, Chans3] = [open_channels(Conn, 5) || Conn <- Conns1], + + %% we've crossed the limit + expect_that_client_connection_is_rejected(Config, 0), + expect_that_client_connection_is_rejected(Config, 0), + expect_that_client_connection_is_rejected(Config, 0), + expect_that_client_channel_is_rejected(Conn1), + + rabbit_ct_helpers:await_condition( + fun () -> + is_process_alive(Conn1) =:= false andalso + is_process_alive(Conn2) andalso + is_process_alive(Conn3) + end), + + %% reach limit again + [Conn4] = open_connections(Config, [{0, Username}]), + Chans4 = [_|_] = open_channels(Conn4, 5), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 3 andalso + count_channels_of_user(Config, Username) =:= 15 + end), + + clear_all_user_limits(Config, Username), + + [Conn5, Conn6, Conn7] = Conns2 = open_connections(Config, [0, 0, 0]), + [Chans5, Chans6, Chans7] = [open_channels(Conn, 5) || Conn <- Conns2], + + close_channels(Chans2 ++ Chans3 ++ Chans4 ++ Chans5 ++ Chans6 ++ Chans7), + rabbit_ct_helpers:await_condition( + fun () -> + count_channels_of_user(Config, Username) =:= 0 + end), + + close_connections([Conn2, Conn3, Conn4, Conn5, Conn6, Conn7]), + ?assertEqual(0, count_connections_of_user(Config, Username)), + + set_user_connection_and_channel_limit(Config, Username, -1, -1). + +single_node_multiple_users_clear_limits(Config) -> + Username1 = <<"guest1">>, + Username2 = <<"guest2">>, + + set_up_user(Config, Username1), + set_up_user(Config, Username2), + + set_user_connection_and_channel_limit(Config, Username1, 0, 0), + set_user_connection_and_channel_limit(Config, Username2, 0, 0), + + ?assertEqual(0, count_connections_of_user(Config, Username1)), + ?assertEqual(0, count_connections_of_user(Config, Username2)), + ?assertEqual(0, count_channels_of_user(Config, Username1)), + ?assertEqual(0, count_channels_of_user(Config, Username2)), + + %% with limit = 0 no connections are allowed + expect_that_client_connection_is_rejected(Config, 0, Username1), + expect_that_client_connection_is_rejected(Config, 0, Username2), + expect_that_client_connection_is_rejected(Config, 0, Username1), + + %% with limit = 0 no channels are allowed + set_user_connection_and_channel_limit(Config, Username1, 1, 0), + set_user_connection_and_channel_limit(Config, Username2, 1, 0), + [ConnA, ConnB] = open_connections(Config, [{0, Username1}, {0, Username2}]), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username1) =:= 1 + end), + expect_that_client_channel_is_rejected(ConnA), + expect_that_client_channel_is_rejected(ConnB), + + rabbit_ct_helpers:await_condition( + fun () -> + is_process_alive(ConnA) =:= false andalso + is_process_alive(ConnB) =:= false + end), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username1) =:= 0 andalso + count_connections_of_user(Config, Username2) =:= 0 + end), + rabbit_ct_helpers:await_condition( + fun () -> + count_channels_of_user(Config, Username1) =:= 0 andalso + count_channels_of_user(Config, Username2) =:= 0 + end), + + clear_all_user_limits(Config, Username1), + set_user_channel_limit_only(Config, Username2, -1), + set_user_connection_limit_only(Config, Username2, -1), + + [Conn1, Conn2] = Conns1 = open_connections(Config, [{0, Username1}, {0, Username1}]), + [Chans1, Chans2] = [open_channels(Conn, 5) || Conn <- Conns1], + + close_channels(Chans1 ++ Chans2), + rabbit_ct_helpers:await_condition( + fun () -> + count_channels_of_user(Config, Username1) =:= 0 andalso + count_channels_of_user(Config, Username2) =:= 0 + end), + + close_connections([Conn1, Conn2]), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username1) =:= 0 andalso + count_connections_of_user(Config, Username2) =:= 0 + end), + + set_user_connection_and_channel_limit(Config, Username1, -1, -1), + set_user_connection_and_channel_limit(Config, Username2, -1, -1). + +single_node_multiple_users_limit(Config) -> + Username1 = <<"guest1">>, + Username2 = <<"guest2">>, + + set_up_user(Config, Username1), + set_up_user(Config, Username2), + + set_user_connection_and_channel_limit(Config, Username1, 2, 10), + set_user_connection_and_channel_limit(Config, Username2, 2, 10), + + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username1) =:= 0 andalso + count_connections_of_user(Config, Username2) =:= 0 + end), + rabbit_ct_helpers:await_condition( + fun () -> + count_channels_of_user(Config, Username1) =:= 0 andalso + count_channels_of_user(Config, Username2) =:= 0 + end), + + [Conn1, Conn2, Conn3, Conn4] = Conns1 = open_connections(Config, [ + {0, Username1}, + {0, Username1}, + {0, Username2}, + {0, Username2}]), + + [_Chans1, Chans2, Chans3, Chans4] = [open_channels(Conn, 5) || Conn <- Conns1], + + %% we've crossed the limit + expect_that_client_connection_is_rejected(Config, 0, Username1), + expect_that_client_connection_is_rejected(Config, 0, Username2), + expect_that_client_channel_is_rejected(Conn1), + rabbit_ct_helpers:await_condition( + fun () -> + is_process_alive(Conn1) =:= false andalso + is_process_alive(Conn3) =:= true + end), + + [Conn5] = open_connections(Config, [0]), + Chans5 = [_|_] = open_channels(Conn5, 5), + + set_user_connection_and_channel_limit(Config, Username1, 5, 25), + set_user_connection_and_channel_limit(Config, Username2, -10, -50), + + [Conn6, Conn7, Conn8, Conn9, Conn10] = Conns2 = open_connections(Config, [ + {0, Username1}, + {0, Username1}, + {0, Username1}, + {0, Username2}, + {0, Username2}]), + + [Chans6, Chans7, Chans8, Chans9, Chans10] = [open_channels(Conn, 5) || Conn <- Conns2], + + close_channels(Chans2 ++ Chans3 ++ Chans4 ++ Chans5 ++ Chans6 ++ + Chans7 ++ Chans8 ++ Chans9 ++ Chans10), + rabbit_ct_helpers:await_condition( + fun () -> + count_channels_of_user(Config, Username1) =:= 0 andalso + count_channels_of_user(Config, Username2) =:= 0 + end), + + close_connections([Conn2, Conn3, Conn4, Conn5, Conn6, + Conn7, Conn8, Conn9, Conn10]), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username1) =:= 0 andalso + count_connections_of_user(Config, Username2) =:= 0 + end), + + set_user_connection_and_channel_limit(Config, Username1, -1, -1), + set_user_connection_and_channel_limit(Config, Username2, -1, -1), + + rabbit_ct_broker_helpers:delete_user(Config, Username1), + rabbit_ct_broker_helpers:delete_user(Config, Username2). + + +single_node_multiple_users_zero_limit(Config) -> + Username1 = <<"guest1">>, + Username2 = <<"guest2">>, + + set_up_user(Config, Username1), + set_up_user(Config, Username2), + + set_user_connection_and_channel_limit(Config, Username1, 0, 0), + set_user_connection_and_channel_limit(Config, Username2, 0, 0), + + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username1) =:= 0 andalso + count_connections_of_user(Config, Username2) =:= 0 + end), + rabbit_ct_helpers:await_condition( + fun () -> + count_channels_of_user(Config, Username1) =:= 0 andalso + count_channels_of_user(Config, Username2) =:= 0 + end), + + %% with limit = 0 no connections are allowed + expect_that_client_connection_is_rejected(Config, 0, Username1), + expect_that_client_connection_is_rejected(Config, 0, Username2), + expect_that_client_connection_is_rejected(Config, 0, Username1), + + %% with limit = 0 no channels are allowed + set_user_connection_and_channel_limit(Config, Username1, 1, 0), + set_user_connection_and_channel_limit(Config, Username2, 1, 0), + [ConnA, ConnB] = open_connections(Config, [{0, Username1}, {0, Username2}]), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username1) =:= 1 + end), + expect_that_client_channel_is_rejected(ConnA), + expect_that_client_channel_is_rejected(ConnB), + + rabbit_ct_helpers:await_condition( + fun () -> + is_process_alive(ConnA) =:= false andalso + is_process_alive(ConnB) =:= false + end), + + ?assertEqual(false, is_process_alive(ConnA)), + ?assertEqual(false, is_process_alive(ConnB)), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username1) =:= 0 andalso + count_connections_of_user(Config, Username2) =:= 0 + end), + rabbit_ct_helpers:await_condition( + fun () -> + count_channels_of_user(Config, Username1) =:= 0 andalso + count_channels_of_user(Config, Username2) =:= 0 + end), + + set_user_connection_and_channel_limit(Config, Username1, -1, -1), + [Conn1, Conn2] = Conns1 = open_connections(Config, [{0, Username1}, {0, Username1}]), + [Chans1, Chans2] = [open_channels(Conn, 5) || Conn <- Conns1], + + close_channels(Chans1 ++ Chans2), + rabbit_ct_helpers:await_condition( + fun () -> + count_channels_of_user(Config, Username1) =:= 0 andalso + count_channels_of_user(Config, Username2) =:= 0 + end), + + close_connections([Conn1, Conn2]), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username1) =:= 0 andalso + count_connections_of_user(Config, Username2) =:= 0 + end), + + set_user_connection_and_channel_limit(Config, Username1, -1, -1), + set_user_connection_and_channel_limit(Config, Username2, -1, -1). + + +cluster_single_user_limit(Config) -> + Username = proplists:get_value(rmq_username, Config), + set_user_connection_limit_only(Config, Username, 2), + set_user_channel_limit_only(Config, Username, 10), + + rabbit_ct_helpers:await_condition( + fun () -> + count_channels_of_user(Config, Username) =:= 0 andalso + count_channels_of_user(Config, Username) =:= 0 + end), + + %% here connections and channels are opened to different nodes + [Conn1, Conn2] = Conns1 = open_connections(Config, [{0, Username}, {1, Username}]), + [_Chans1, Chans2] = [open_channels(Conn, 5) || Conn <- Conns1], + + %% we've crossed the limit + expect_that_client_connection_is_rejected(Config, 0, Username), + expect_that_client_connection_is_rejected(Config, 1, Username), + expect_that_client_channel_is_rejected(Conn1), + rabbit_ct_helpers:await_condition( + fun () -> + is_process_alive(Conn1) =:= false andalso + is_process_alive(Conn2) =:= true + end), + + set_user_connection_and_channel_limit(Config, Username, 5, 25), + + [Conn3, Conn4] = Conns2 = open_connections(Config, [{0, Username}, {0, Username}]), + [Chans3, Chans4] = [open_channels(Conn, 5) || Conn <- Conns2], + + close_channels(Chans2 ++ Chans3 ++ Chans4), + ?assertEqual(0, count_channels_of_user(Config, Username)), + + close_connections([Conn2, Conn3, Conn4]), + ?assertEqual(0, count_connections_of_user(Config, Username)), + + set_user_connection_and_channel_limit(Config, Username, -1, -1). + +cluster_single_user_limit2(Config) -> + Username = proplists:get_value(rmq_username, Config), + set_user_connection_and_channel_limit(Config, Username, 2, 10), + + ?assertEqual(0, count_connections_of_user(Config, Username)), + ?assertEqual(0, count_channels_of_user(Config, Username)), + + %% here a limit is reached on one node first + [Conn1, Conn2] = Conns1 = open_connections(Config, [{0, Username}, {0, Username}]), + [_Chans1, Chans2] = [open_channels(Conn, 5) || Conn <- Conns1], + + %% we've crossed the limit + expect_that_client_connection_is_rejected(Config, 0, Username), + expect_that_client_connection_is_rejected(Config, 1, Username), + expect_that_client_channel_is_rejected(Conn1), + rabbit_ct_helpers:await_condition( + fun () -> + is_process_alive(Conn1) =:= false andalso + is_process_alive(Conn2) =:= true + end), + + set_user_connection_and_channel_limit(Config, Username, 5, 25), + + [Conn3, Conn4, Conn5, Conn6, {error, not_allowed}] = open_connections(Config, [ + {1, Username}, + {1, Username}, + {1, Username}, + {1, Username}, + {1, Username}]), + + [Chans3, Chans4, Chans5, Chans6, [{error, not_allowed}]] = + [open_channels(Conn, 1) || Conn <- [Conn3, Conn4, Conn5, Conn6, Conn1]], + + close_channels(Chans2 ++ Chans3 ++ Chans4 ++ Chans5 ++ Chans6), + rabbit_ct_helpers:await_condition( + fun () -> + count_channels_of_user(Config, Username) =:= 0 + end), + + close_connections([Conn2, Conn3, Conn4, Conn5, Conn6]), + ?assertEqual(0, count_connections_of_user(Config, Username)), + + set_user_connection_and_channel_limit(Config, Username, -1, -1). + + +cluster_single_user_zero_limit(Config) -> + Username = proplists:get_value(rmq_username, Config), + set_user_connection_and_channel_limit(Config, Username, 0, 0), + + ?assertEqual(0, count_connections_of_user(Config, Username)), + ?assertEqual(0, count_channels_of_user(Config, Username)), + + %% with limit = 0 no connections are allowed + expect_that_client_connection_is_rejected(Config, 0), + expect_that_client_connection_is_rejected(Config, 1), + expect_that_client_connection_is_rejected(Config, 0), + + %% with limit = 0 no channels are allowed + set_user_connection_and_channel_limit(Config, Username, 1, 0), + [ConnA] = open_connections(Config, [0]), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 1 + end), + expect_that_client_channel_is_rejected(ConnA), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 0 andalso + count_channels_of_user(Config, Username) =:= 0 + end), + ?assertEqual(false, is_process_alive(ConnA)), + + set_user_connection_and_channel_limit(Config, Username, -1, -1), + [Conn1, Conn2, Conn3, Conn4] = Conns1 = open_connections(Config, [0, 1, 0, 1]), + [Chans1, Chans2, Chans3, Chans4] = [open_channels(Conn, 5) || Conn <- Conns1], + + close_channels(Chans1 ++ Chans2 ++ Chans3 ++ Chans4), + rabbit_ct_helpers:await_condition( + fun () -> + count_channels_of_user(Config, Username) =:= 0 + end), + + close_connections([Conn1, Conn2, Conn3, Conn4]), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 0 + end), + + set_user_connection_and_channel_limit(Config, Username, -1, -1). + +cluster_single_user_clear_limits(Config) -> + Username = proplists:get_value(rmq_username, Config), + set_user_connection_and_channel_limit(Config, Username, 2, 10), + + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 0 andalso + count_channels_of_user(Config, Username) =:= 0 + end), + + %% here a limit is reached on one node first + [Conn1, Conn2] = Conns1 = open_connections(Config, [{0, Username}, {0, Username}]), + [_Chans1, Chans2] = [open_channels(Conn, 5) || Conn <- Conns1], + + %% we've crossed the limit + expect_that_client_connection_is_rejected(Config, 0, Username), + expect_that_client_connection_is_rejected(Config, 1, Username), + expect_that_client_channel_is_rejected(Conn1), + rabbit_ct_helpers:await_condition( + fun () -> + is_process_alive(Conn1) =:= false andalso + is_process_alive(Conn2) =:= true + end), + clear_all_user_limits(Config, Username), + + [Conn3, Conn4, Conn5, Conn6, Conn7] = open_connections(Config, [ + {1, Username}, + {1, Username}, + {1, Username}, + {1, Username}, + {1, Username}]), + + [Chans3, Chans4, Chans5, Chans6, Chans7] = + [open_channels(Conn, 1) || Conn <- [Conn3, Conn4, Conn5, Conn6, Conn7]], + + close_channels(Chans2 ++ Chans3 ++ Chans4 ++ Chans5 ++ Chans6 ++ Chans7), + rabbit_ct_helpers:await_condition( + fun () -> + count_channels_of_user(Config, Username) =:= 0 + end), + + close_connections([Conn2, Conn3, Conn4, Conn5, Conn6, Conn7]), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 0 + end), + + set_user_connection_and_channel_limit(Config, Username, -1, -1). + +cluster_multiple_users_clear_limits(Config) -> + Username1 = <<"guest1">>, + Username2 = <<"guest2">>, + + set_up_user(Config, Username1), + set_up_user(Config, Username2), + + set_user_connection_and_channel_limit(Config, Username1, 0, 0), + set_user_connection_and_channel_limit(Config, Username2, 0, 0), + + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username1) =:= 0 andalso + count_connections_of_user(Config, Username2) =:= 0 + end), + rabbit_ct_helpers:await_condition( + fun () -> + count_channels_of_user(Config, Username1) =:= 0 andalso + count_channels_of_user(Config, Username2) =:= 0 + end), + + %% with limit = 0 no connections are allowed + expect_that_client_connection_is_rejected(Config, 0, Username1), + expect_that_client_connection_is_rejected(Config, 0, Username2), + expect_that_client_connection_is_rejected(Config, 1, Username1), + expect_that_client_connection_is_rejected(Config, 1, Username2), + + %% with limit = 0 no channels are allowed + set_user_connection_and_channel_limit(Config, Username1, 1, 0), + set_user_connection_and_channel_limit(Config, Username2, 1, 0), + [ConnA, ConnB] = open_connections(Config, [{0, Username1}, {1, Username2}]), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username1) =:= 1 andalso + count_connections_of_user(Config, Username2) =:= 1 + end), + expect_that_client_channel_is_rejected(ConnA), + + rabbit_ct_helpers:await_condition( + fun () -> + is_process_alive(ConnA) =:= false andalso + is_process_alive(ConnB) =:= true + end), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username1) =:= 0 andalso + count_connections_of_user(Config, Username2) =:= 1 + end), + rabbit_ct_helpers:await_condition( + fun () -> + count_channels_of_user(Config, Username1) =:= 0 andalso + count_channels_of_user(Config, Username2) =:= 0 + end), + close_connections([ConnB]), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username2) =:= 0 andalso + count_channels_of_user(Config, Username2) =:= 0 + end), + ?assertEqual(false, is_process_alive(ConnB)), + + clear_all_user_limits(Config, Username1), + clear_all_user_limits(Config, Username2), + + [Conn1, Conn2, Conn3, Conn4] = Conns1 = open_connections(Config, [ + {0, Username1}, + {0, Username2}, + {1, Username1}, + {1, Username2}]), + + [Chans1, Chans2, Chans3, Chans4] = [open_channels(Conn, 5) || Conn <- Conns1], + + close_channels(Chans1 ++ Chans2 ++ Chans3 ++ Chans4), + rabbit_ct_helpers:await_condition( + fun () -> + count_channels_of_user(Config, Username1) =:= 0 andalso + count_channels_of_user(Config, Username2) =:= 0 + end), + + close_connections([Conn1, Conn2, Conn3, Conn4]), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username1) =:= 0 andalso + count_connections_of_user(Config, Username2) =:= 0 + end), + + set_user_connection_and_channel_limit(Config, Username1, -1, -1), + set_user_connection_and_channel_limit(Config, Username2, -1, -1). + +cluster_multiple_users_zero_limit(Config) -> + Username1 = <<"guest1">>, + Username2 = <<"guest2">>, + + set_up_user(Config, Username1), + set_up_user(Config, Username2), + + set_user_connection_and_channel_limit(Config, Username1, 0, 0), + set_user_connection_and_channel_limit(Config, Username2, 0, 0), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username1) =:= 0 andalso + count_connections_of_user(Config, Username2) =:= 0 + end), + rabbit_ct_helpers:await_condition( + fun () -> + count_channels_of_user(Config, Username1) =:= 0 andalso + count_channels_of_user(Config, Username2) =:= 0 + end), + + %% with limit = 0 no connections are allowed + expect_that_client_connection_is_rejected(Config, 0, Username1), + expect_that_client_connection_is_rejected(Config, 0, Username2), + expect_that_client_connection_is_rejected(Config, 1, Username1), + expect_that_client_connection_is_rejected(Config, 1, Username2), + + %% with limit = 0 no channels are allowed + set_user_connection_and_channel_limit(Config, Username1, 1, 0), + set_user_connection_and_channel_limit(Config, Username2, 1, 0), + [ConnA, ConnB] = open_connections(Config, [{0, Username1}, {1, Username2}]), + + expect_that_client_channel_is_rejected(ConnA), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username1) =:= 0 andalso + count_connections_of_user(Config, Username2) =:= 1 + end), + rabbit_ct_helpers:await_condition( + fun () -> + count_channels_of_user(Config, Username1) =:= 0 andalso + count_channels_of_user(Config, Username2) =:= 0 + end), + ?assertEqual(false, is_process_alive(ConnA)), + ?assertEqual(true, is_process_alive(ConnB)), + close_connections([ConnB]), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username2) =:= 0 andalso + count_channels_of_user(Config, Username2) =:= 0 + end), + ?assertEqual(false, is_process_alive(ConnB)), + + set_user_connection_and_channel_limit(Config, Username1, -1, -1), + set_user_connection_and_channel_limit(Config, Username2, -1, -1), + + [Conn1, Conn2, Conn3, Conn4] = Conns1 = open_connections(Config, [ + {0, Username1}, + {0, Username2}, + {1, Username1}, + {1, Username2}]), + + [Chans1, Chans2, Chans3, Chans4] = [open_channels(Conn, 5) || Conn <- Conns1], + + close_channels(Chans1 ++ Chans2 ++ Chans3 ++ Chans4), + ?assertEqual(0, count_channels_of_user(Config, Username1)), + ?assertEqual(0, count_channels_of_user(Config, Username2)), + + close_connections([Conn1, Conn2, Conn3, Conn4]), + ?assertEqual(0, count_connections_of_user(Config, Username1)), + ?assertEqual(0, count_connections_of_user(Config, Username2)), + + set_user_connection_and_channel_limit(Config, Username1, -1, -1), + set_user_connection_and_channel_limit(Config, Username2, -1, -1). + +%% ------------------------------------------------------------------- +%% Helpers +%% ------------------------------------------------------------------- + +open_connections(Config, NodesAndUsers) -> + % Randomly select connection type + OpenConnectionFun = case ?config(connection_type, Config) of + network -> open_unmanaged_connection; + direct -> open_unmanaged_connection_direct + end, + Conns = lists:map(fun + ({Node, User}) -> + rabbit_ct_client_helpers:OpenConnectionFun(Config, Node, + User, User); + (Node) -> + rabbit_ct_client_helpers:OpenConnectionFun(Config, Node) + end, NodesAndUsers), + timer:sleep(100), + Conns. + +close_connections(Conns) -> + lists:foreach(fun + (Conn) -> + rabbit_ct_client_helpers:close_connection(Conn) + end, Conns). + +open_channels(Conn, N) -> + [open_channel(Conn) || _ <- lists:seq(1, N)]. + +open_channel(Conn) when is_pid(Conn) -> + try amqp_connection:open_channel(Conn) of + {ok, Ch} -> Ch + catch + _:_Error -> {error, not_allowed} + end. + +close_channels(Channels = [_|_]) -> + [rabbit_ct_client_helpers:close_channel(Ch) || Ch <- Channels]. + +count_connections_of_user(Config, Username) -> + count_connections_in(Config, Username, 0). +count_connections_in(Config, Username, NodeIndex) -> + count_user_tracked_items(Config, NodeIndex, rabbit_connection_tracking, Username). + + count_channels_of_user(Config, Username) -> + count_channels_in(Config, Username, 0). + count_channels_in(Config, Username, NodeIndex) -> + count_user_tracked_items(Config, NodeIndex, rabbit_channel_tracking, Username). + +count_user_tracked_items(Config, NodeIndex, TrackingMod, Username) -> + rabbit_ct_broker_helpers:rpc(Config, NodeIndex, + TrackingMod, + count_tracked_items_in, [{user, Username}]). + +connections_in(Config, Username) -> + connections_in(Config, 0, Username). +connections_in(Config, NodeIndex, Username) -> + tracked_list_of_user(Config, NodeIndex, rabbit_connection_tracking, Username). + +channels_in(Config, Username) -> + channels_in(Config, 0, Username). +channels_in(Config, NodeIndex, Username) -> + tracked_list_of_user(Config, NodeIndex, rabbit_channel_tracking, Username). + +tracked_list_of_user(Config, NodeIndex, TrackingMod, Username) -> + rabbit_ct_broker_helpers:rpc(Config, NodeIndex, + TrackingMod, + list_of_user, [Username]). + +connections_on_node(Config) -> + connections_on_node(Config, 0). +connections_on_node(Config, NodeIndex) -> + Node = rabbit_ct_broker_helpers:get_node_config(Config, NodeIndex, nodename), + tracked_items_on_node(Config, NodeIndex, rabbit_connection_tracking, Node). + +channels_on_node(Config) -> + channels_on_node(Config, 0). +channels_on_node(Config, NodeIndex) -> + Node = rabbit_ct_broker_helpers:get_node_config(Config, NodeIndex, nodename), + tracked_items_on_node(Config, NodeIndex, rabbit_channel_tracking, Node). + +tracked_items_on_node(Config, NodeIndex, TrackingMod, NodeForListing) -> + rabbit_ct_broker_helpers:rpc(Config, NodeIndex, + TrackingMod, + list_on_node, [NodeForListing]). + +all_connections(Config) -> + all_connections(Config, 0). +all_connections(Config, NodeIndex) -> + all_tracked_items(Config, NodeIndex, rabbit_connection_tracking). + +all_channels(Config) -> + all_channels(Config, 0). +all_channels(Config, NodeIndex) -> + all_tracked_items(Config, NodeIndex, rabbit_channel_tracking). + +all_tracked_items(Config, NodeIndex, TrackingMod) -> + rabbit_ct_broker_helpers:rpc(Config, NodeIndex, + TrackingMod, + list, []). + +set_up_user(Config, Username) -> + VHost = proplists:get_value(rmq_vhost, Config), + rabbit_ct_broker_helpers:add_user(Config, Username), + rabbit_ct_broker_helpers:set_full_permissions(Config, Username, VHost), + set_user_connection_and_channel_limit(Config, Username, -1, -1). + +set_user_connection_and_channel_limit(Config, Username, ConnLimit, ChLimit) -> + set_user_connection_and_channel_limit(Config, 0, Username, ConnLimit, ChLimit). + +set_user_connection_and_channel_limit(Config, NodeIndex, Username, ConnLimit, ChLimit) -> + Node = rabbit_ct_broker_helpers:get_node_config( + Config, NodeIndex, nodename), + ok = rabbit_ct_broker_helpers:control_action( + set_user_limits, Node, [rabbit_data_coercion:to_list(Username)] ++ + ["{\"max-connections\": " ++ integer_to_list(ConnLimit) ++ "," ++ + " \"max-channels\": " ++ integer_to_list(ChLimit) ++ "}"]). + +set_user_connection_limit_only(Config, Username, ConnLimit) -> + set_user_connection_limit_only(Config, 0, Username, ConnLimit). + +set_user_connection_limit_only(Config, NodeIndex, Username, ConnLimit) -> + Node = rabbit_ct_broker_helpers:get_node_config( + Config, NodeIndex, nodename), + ok = rabbit_ct_broker_helpers:control_action( + set_user_limits, Node, [rabbit_data_coercion:to_list(Username)] ++ + ["{\"max-connections\": " ++ integer_to_list(ConnLimit) ++ "}"]). + +set_user_channel_limit_only(Config, Username, ChLimit) -> + set_user_channel_limit_only(Config, 0, Username, ChLimit). + +set_user_channel_limit_only(Config, NodeIndex, Username, ChLimit) -> + Node = rabbit_ct_broker_helpers:get_node_config( + Config, NodeIndex, nodename), + ok = rabbit_ct_broker_helpers:control_action( + set_user_limits, Node, [rabbit_data_coercion:to_list(Username)] ++ + ["{\"max-channels\": " ++ integer_to_list(ChLimit) ++ "}"]). + +clear_all_user_limits(Config, Username) -> + clear_all_user_limits(Config, 0, Username). +clear_all_user_limits(Config, NodeIndex, Username) -> + Node = rabbit_ct_broker_helpers:get_node_config( + Config, NodeIndex, nodename), + ok = rabbit_ct_broker_helpers:control_action( + clear_user_limits, Node, [rabbit_data_coercion:to_list(Username), "all"]). + +await_running_node_refresh(_Config, _NodeIndex) -> + timer:sleep(250). + +expect_that_client_connection_is_rejected(Config) -> + expect_that_client_connection_is_rejected(Config, 0). + +expect_that_client_connection_is_rejected(Config, NodeIndex) -> + {error, not_allowed} = + rabbit_ct_client_helpers:open_unmanaged_connection(Config, NodeIndex). + +expect_that_client_connection_is_rejected(Config, NodeIndex, User) -> + {error, not_allowed} = + rabbit_ct_client_helpers:open_unmanaged_connection(Config, NodeIndex, User, User). + +expect_that_client_channel_is_rejected(Conn) -> + {error, not_allowed} = open_channel(Conn). diff --git a/test/per_user_connection_channel_limit_partitions_SUITE.erl b/test/per_user_connection_channel_limit_partitions_SUITE.erl new file mode 100644 index 0000000000..32af9ce9a1 --- /dev/null +++ b/test/per_user_connection_channel_limit_partitions_SUITE.erl @@ -0,0 +1,174 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(per_user_connection_channel_limit_partitions_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-compile(export_all). + +-import(rabbit_ct_client_helpers, [open_unmanaged_connection/2, + open_unmanaged_connection/3]). + +all() -> + [ + {group, net_ticktime_1} + ]. + +groups() -> + [ + {net_ticktime_1, [], [ + cluster_full_partition_with_autoheal + ]} + ]. + +suite() -> + [ + %% If a test hangs, no need to wait for 30 minutes. + {timetrap, {minutes, 8}} + ]. + +%% see partitions_SUITE +-define(DELAY, 12000). + +%% ------------------------------------------------------------------- +%% Testsuite setup/teardown. +%% ------------------------------------------------------------------- + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config, [ + fun rabbit_ct_broker_helpers:configure_dist_proxy/1 + ]). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(net_ticktime_1 = Group, Config) -> + Config1 = rabbit_ct_helpers:set_config(Config, [{net_ticktime, 1}]), + init_per_multinode_group(Group, Config1, 3). + +init_per_multinode_group(_Group, Config, NodeCount) -> + Suffix = rabbit_ct_helpers:testcase_absname(Config, "", "-"), + Config1 = rabbit_ct_helpers:set_config(Config, [ + {rmq_nodes_count, NodeCount}, + {rmq_nodename_suffix, Suffix} + ]), + rabbit_ct_helpers:run_steps(Config1, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()). + +end_per_group(_Group, Config) -> + rabbit_ct_helpers:run_steps(Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()). + +init_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_started(Config, Testcase). + +end_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_finished(Config, Testcase). + +%% ------------------------------------------------------------------- +%% Test cases. +%% ------------------------------------------------------------------- + +cluster_full_partition_with_autoheal(Config) -> + Username = proplists:get_value(rmq_username, Config), + rabbit_ct_broker_helpers:set_partition_handling_mode_globally(Config, autoheal), + + ?assertEqual(0, count_connections_in(Config, Username)), + [A, B, C] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + %% 6 connections, 2 per node + Conn1 = open_unmanaged_connection(Config, A), + Conn2 = open_unmanaged_connection(Config, A), + Conn3 = open_unmanaged_connection(Config, B), + Conn4 = open_unmanaged_connection(Config, B), + Conn5 = open_unmanaged_connection(Config, C), + Conn6 = open_unmanaged_connection(Config, C), + + _Chans1 = [_|_] = open_channels(Conn1, 5), + _Chans3 = [_|_] = open_channels(Conn3, 5), + _Chans5 = [_|_] = open_channels(Conn5, 5), + wait_for_count_connections_in(Config, Username, 6, 60000), + ?assertEqual(15, count_channels_in(Config, Username)), + + %% B drops off the network, non-reachable by either A or C + rabbit_ct_broker_helpers:block_traffic_between(A, B), + rabbit_ct_broker_helpers:block_traffic_between(B, C), + timer:sleep(?DELAY), + + %% A and C are still connected, so 4 connections are tracked + %% All connections to B are dropped + wait_for_count_connections_in(Config, Username, 4, 60000), + ?assertEqual(10, count_channels_in(Config, Username)), + + rabbit_ct_broker_helpers:allow_traffic_between(A, B), + rabbit_ct_broker_helpers:allow_traffic_between(B, C), + timer:sleep(?DELAY), + + %% during autoheal B's connections were dropped + wait_for_count_connections_in(Config, Username, 4, 60000), + ?assertEqual(10, count_channels_in(Config, Username)), + + lists:foreach(fun (Conn) -> + (catch rabbit_ct_client_helpers:close_connection(Conn)) + end, [Conn1, Conn2, Conn3, Conn4, + Conn5, Conn6]), + ?assertEqual(0, count_connections_in(Config, Username)), + ?assertEqual(0, count_channels_in(Config, Username)), + + passed. + +%% ------------------------------------------------------------------- +%% Helpers +%% ------------------------------------------------------------------- + +wait_for_count_connections_in(Config, Username, Expected, Time) when Time =< 0 -> + ?assertMatch(Connections when length(Connections) == Expected, + connections_in(Config, Username)); +wait_for_count_connections_in(Config, Username, Expected, Time) -> + case connections_in(Config, Username) of + Connections when length(Connections) == Expected -> + ok; + _ -> + Sleep = 3000, + timer:sleep(Sleep), + wait_for_count_connections_in(Config, Username, Expected, Time - Sleep) + end. + +open_channels(Conn, N) -> + [begin + {ok, Ch} = amqp_connection:open_channel(Conn), + Ch + end || _ <- lists:seq(1, N)]. + +count_connections_in(Config, Username) -> + length(connections_in(Config, Username)). + +connections_in(Config, Username) -> + connections_in(Config, 0, Username). +connections_in(Config, NodeIndex, Username) -> + tracked_list_of_user(Config, NodeIndex, rabbit_connection_tracking, Username). + +count_channels_in(Config, Username) -> + Channels = channels_in(Config, Username), + length([Ch || Ch = #tracked_channel{username = Username0} <- Channels, + Username =:= Username0]). + +channels_in(Config, Username) -> + channels_in(Config, 0, Username). +channels_in(Config, NodeIndex, Username) -> + tracked_list_of_user(Config, NodeIndex, rabbit_channel_tracking, Username). + +tracked_list_of_user(Config, NodeIndex, TrackingMod, Username) -> + rabbit_ct_broker_helpers:rpc(Config, NodeIndex, + TrackingMod, + list_of_user, [Username]). diff --git a/test/per_user_connection_channel_tracking_SUITE.erl b/test/per_user_connection_channel_tracking_SUITE.erl new file mode 100644 index 0000000000..674a2e4177 --- /dev/null +++ b/test/per_user_connection_channel_tracking_SUITE.erl @@ -0,0 +1,840 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(per_user_connection_channel_tracking_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-compile(export_all). + +all() -> + [ + {group, cluster_size_1_network}, + {group, cluster_size_2_network}, + {group, cluster_size_1_direct}, + {group, cluster_size_2_direct} + ]. + +groups() -> + ClusterSize1Tests = [ + single_node_user_connection_channel_tracking, + single_node_user_deletion, + single_node_vhost_down_mimic, + single_node_vhost_deletion + ], + ClusterSize2Tests = [ + cluster_user_deletion, + cluster_vhost_down_mimic, + cluster_vhost_deletion, + cluster_node_removed + ], + [ + {cluster_size_1_network, [], ClusterSize1Tests}, + {cluster_size_2_network, [], ClusterSize2Tests}, + {cluster_size_1_direct, [], ClusterSize1Tests}, + {cluster_size_2_direct, [], ClusterSize2Tests} + ]. + +suite() -> + [ + %% If a test hangs, no need to wait for 30 minutes. + {timetrap, {minutes, 8}} + ]. + +%% ------------------------------------------------------------------- +%% Testsuite setup/teardown. +%% ------------------------------------------------------------------- + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(cluster_size_1_network, Config) -> + Config1 = rabbit_ct_helpers:set_config(Config, [{connection_type, network}]), + init_per_multinode_group(cluster_size_1_network, Config1, 1); +init_per_group(cluster_size_2_network, Config) -> + Config1 = rabbit_ct_helpers:set_config(Config, [{connection_type, network}]), + init_per_multinode_group(cluster_size_2_network, Config1, 2); +init_per_group(cluster_size_1_direct, Config) -> + Config1 = rabbit_ct_helpers:set_config(Config, [{connection_type, direct}]), + init_per_multinode_group(cluster_size_1_direct, Config1, 1); +init_per_group(cluster_size_2_direct, Config) -> + Config1 = rabbit_ct_helpers:set_config(Config, [{connection_type, direct}]), + init_per_multinode_group(cluster_size_2_direct, Config1, 2). + +init_per_multinode_group(_Group, Config, NodeCount) -> + Suffix = rabbit_ct_helpers:testcase_absname(Config, "", "-"), + Config1 = rabbit_ct_helpers:set_config(Config, [ + {rmq_nodes_count, NodeCount}, + {rmq_nodename_suffix, Suffix} + ]), + rabbit_ct_helpers:run_steps(Config1, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()). + +end_per_group(_Group, Config) -> + rabbit_ct_helpers:run_steps(Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()). + +init_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_started(Config, Testcase), + clear_all_connection_tracking_tables(Config), + clear_all_channel_tracking_tables(Config), + Config. + +end_per_testcase(Testcase, Config) -> + clear_all_connection_tracking_tables(Config), + clear_all_channel_tracking_tables(Config), + rabbit_ct_helpers:testcase_finished(Config, Testcase). + +clear_all_connection_tracking_tables(Config) -> + [rabbit_ct_broker_helpers:rpc(Config, + N, + rabbit_connection_tracking, + clear_tracking_tables, + []) || N <- rabbit_ct_broker_helpers:get_node_configs(Config, nodename)]. + +clear_all_channel_tracking_tables(Config) -> + [rabbit_ct_broker_helpers:rpc(Config, + N, + rabbit_channel_tracking, + clear_tracking_tables, + []) || N <- rabbit_ct_broker_helpers:get_node_configs(Config, nodename)]. + +%% ------------------------------------------------------------------- +%% Test cases. +%% ------------------------------------------------------------------- +single_node_user_connection_channel_tracking(Config) -> + Username = proplists:get_value(rmq_username, Config), + Username2 = <<"guest2">>, + + Vhost = proplists:get_value(rmq_vhost, Config), + + rabbit_ct_broker_helpers:add_user(Config, Username2), + rabbit_ct_broker_helpers:set_full_permissions(Config, Username2, Vhost), + + ?assertEqual(0, count_connections_in(Config, Username)), + ?assertEqual(0, count_connections_in(Config, Username2)), + ?assertEqual(0, count_channels_in(Config, Username)), + ?assertEqual(0, count_channels_in(Config, Username2)), + ?assertEqual(0, tracked_user_connection_count(Config, Username)), + ?assertEqual(0, tracked_user_connection_count(Config, Username2)), + ?assertEqual(0, tracked_user_channel_count(Config, Username)), + ?assertEqual(0, tracked_user_channel_count(Config, Username2)), + + [Conn1] = open_connections(Config, [0]), + [Chan1] = open_channels(Conn1, 1), + [#tracked_connection{username = Username}] = connections_in(Config, Username), + [#tracked_channel{username = Username}] = channels_in(Config, Username), + ?assertEqual(true, is_process_alive(Conn1)), + ?assertEqual(true, is_process_alive(Chan1)), + close_channels([Chan1]), + ?assertEqual(0, count_channels_in(Config, Username)), + ?assertEqual(0, tracked_user_channel_count(Config, Username)), + ?assertEqual(false, is_process_alive(Chan1)), + close_connections([Conn1]), + ?assertEqual(0, length(connections_in(Config, Username))), + ?assertEqual(0, tracked_user_connection_count(Config, Username)), + ?assertEqual(false, is_process_alive(Conn1)), + + [Conn2] = open_connections(Config, [{0, Username2}]), + Chans2 = [_|_] = open_channels(Conn2, 5), + timer:sleep(100), + [#tracked_connection{username = Username2}] = connections_in(Config, Username2), + ?assertEqual(5, count_channels_in(Config, Username2)), + ?assertEqual(1, tracked_user_connection_count(Config, Username2)), + ?assertEqual(5, tracked_user_channel_count(Config, Username2)), + ?assertEqual(true, is_process_alive(Conn2)), + [?assertEqual(true, is_process_alive(Ch)) || Ch <- Chans2], + + [Conn3] = open_connections(Config, [0]), + Chans3 = [_|_] = open_channels(Conn3, 5), + [#tracked_connection{username = Username}] = connections_in(Config, Username), + ?assertEqual(5, count_channels_in(Config, Username)), + ?assertEqual(1, tracked_user_connection_count(Config, Username)), + ?assertEqual(5, tracked_user_channel_count(Config, Username)), + ?assertEqual(true, is_process_alive(Conn3)), + [?assertEqual(true, is_process_alive(Ch)) || Ch <- Chans3], + + [Conn4] = open_connections(Config, [0]), + Chans4 = [_|_] = open_channels(Conn4, 5), + ?assertEqual(2, tracked_user_connection_count(Config, Username)), + ?assertEqual(10, tracked_user_channel_count(Config, Username)), + ?assertEqual(true, is_process_alive(Conn4)), + [?assertEqual(true, is_process_alive(Ch)) || Ch <- Chans4], + kill_connections([Conn4]), + [#tracked_connection{username = Username}] = connections_in(Config, Username), + ?assertEqual(5, count_channels_in(Config, Username)), + ?assertEqual(1, tracked_user_connection_count(Config, Username)), + ?assertEqual(5, tracked_user_channel_count(Config, Username)), + ?assertEqual(false, is_process_alive(Conn4)), + [?assertEqual(false, is_process_alive(Ch)) || Ch <- Chans4], + + [Conn5] = open_connections(Config, [0]), + Chans5 = [_|_] = open_channels(Conn5, 7), + [Username, Username] = + lists:map(fun (#tracked_connection{username = U}) -> U end, + connections_in(Config, Username)), + ?assertEqual(12, count_channels_in(Config, Username)), + ?assertEqual(12, tracked_user_channel_count(Config, Username)), + ?assertEqual(2, tracked_user_connection_count(Config, Username)), + ?assertEqual(true, is_process_alive(Conn5)), + [?assertEqual(true, is_process_alive(Ch)) || Ch <- Chans5], + + close_channels(Chans2 ++ Chans3 ++ Chans5), + ?assertEqual(0, length(all_channels(Config))), + ?assertEqual(0, tracked_user_channel_count(Config, Username)), + ?assertEqual(0, tracked_user_channel_count(Config, Username2)), + + close_connections([Conn2, Conn3, Conn5]), + rabbit_ct_broker_helpers:delete_user(Config, Username2), + ?assertEqual(0, tracked_user_connection_count(Config, Username)), + ?assertEqual(0, tracked_user_connection_count(Config, Username2)), + ?assertEqual(0, length(all_connections(Config))). + +single_node_user_deletion(Config) -> + set_tracking_execution_timeout(Config, 100), + + Username = proplists:get_value(rmq_username, Config), + Username2 = <<"guest2">>, + + Vhost = proplists:get_value(rmq_vhost, Config), + + rabbit_ct_broker_helpers:add_user(Config, Username2), + rabbit_ct_broker_helpers:set_full_permissions(Config, Username2, Vhost), + + ?assertEqual(100, get_tracking_execution_timeout(Config)), + + ?assertEqual(0, count_connections_in(Config, Username)), + ?assertEqual(0, count_connections_in(Config, Username2)), + ?assertEqual(0, count_channels_in(Config, Username)), + ?assertEqual(0, count_channels_in(Config, Username2)), + ?assertEqual(0, tracked_user_connection_count(Config, Username)), + ?assertEqual(0, tracked_user_connection_count(Config, Username2)), + ?assertEqual(0, tracked_user_channel_count(Config, Username)), + ?assertEqual(0, tracked_user_channel_count(Config, Username2)), + + [Conn1] = open_connections(Config, [0]), + Chans1 = [_|_] = open_channels(Conn1, 5), + ?assertEqual(1, count_connections_in(Config, Username)), + ?assertEqual(5, count_channels_in(Config, Username)), + ?assertEqual(1, tracked_user_connection_count(Config, Username)), + ?assertEqual(5, tracked_user_channel_count(Config, Username)), + ?assertEqual(true, is_process_alive(Conn1)), + [?assertEqual(true, is_process_alive(Ch)) || Ch <- Chans1], + + [Conn2] = open_connections(Config, [{0, Username2}]), + Chans2 = [_|_] = open_channels(Conn2, 5), + ?assertEqual(1, count_connections_in(Config, Username2)), + ?assertEqual(5, count_channels_in(Config, Username2)), + ?assertEqual(1, tracked_user_connection_count(Config, Username2)), + ?assertEqual(5, tracked_user_channel_count(Config, Username2)), + ?assertEqual(true, is_process_alive(Conn2)), + [?assertEqual(true, is_process_alive(Ch)) || Ch <- Chans2], + + ?assertEqual(true, exists_in_tracked_connection_per_user_table(Config, Username2)), + ?assertEqual(true, exists_in_tracked_channel_per_user_table(Config, Username2)), + + rabbit_ct_broker_helpers:delete_user(Config, Username2), + timer:sleep(100), + ?assertEqual(0, count_connections_in(Config, Username2)), + ?assertEqual(0, count_channels_in(Config, Username2)), + ?assertEqual(0, tracked_user_connection_count(Config, Username2)), + ?assertEqual(0, tracked_user_channel_count(Config, Username2)), + ?assertEqual(false, is_process_alive(Conn2)), + [?assertEqual(false, is_process_alive(Ch)) || Ch <- Chans2], + + %% ensure vhost entry is cleared after 'tracking_execution_timeout' + ?assertEqual(false, exists_in_tracked_connection_per_user_table(Config, Username2)), + ?assertEqual(false, exists_in_tracked_channel_per_user_table(Config, Username2)), + + ?assertEqual(1, count_connections_in(Config, Username)), + ?assertEqual(5, count_channels_in(Config, Username)), + ?assertEqual(1, tracked_user_connection_count(Config, Username)), + ?assertEqual(5, tracked_user_channel_count(Config, Username)), + ?assertEqual(true, is_process_alive(Conn1)), + [?assertEqual(true, is_process_alive(Ch)) || Ch <- Chans1], + + close_channels(Chans1), + ?assertEqual(0, count_channels_in(Config, Username)), + ?assertEqual(0, tracked_user_channel_count(Config, Username)), + + close_connections([Conn1]), + ?assertEqual(0, count_connections_in(Config, Username)), + ?assertEqual(0, tracked_user_connection_count(Config, Username)). + +single_node_vhost_deletion(Config) -> + set_tracking_execution_timeout(Config, 100), + + Username = proplists:get_value(rmq_username, Config), + Username2 = <<"guest2">>, + + Vhost = proplists:get_value(rmq_vhost, Config), + + rabbit_ct_broker_helpers:add_user(Config, Username2), + rabbit_ct_broker_helpers:set_full_permissions(Config, Username2, Vhost), + + ?assertEqual(100, get_tracking_execution_timeout(Config)), + + ?assertEqual(0, count_connections_in(Config, Username)), + ?assertEqual(0, count_connections_in(Config, Username2)), + ?assertEqual(0, count_channels_in(Config, Username)), + ?assertEqual(0, count_channels_in(Config, Username2)), + ?assertEqual(0, tracked_user_connection_count(Config, Username)), + ?assertEqual(0, tracked_user_connection_count(Config, Username2)), + ?assertEqual(0, tracked_user_channel_count(Config, Username)), + ?assertEqual(0, tracked_user_channel_count(Config, Username2)), + + [Conn1] = open_connections(Config, [0]), + Chans1 = [_|_] = open_channels(Conn1, 5), + ?assertEqual(1, count_connections_in(Config, Username)), + ?assertEqual(5, count_channels_in(Config, Username)), + ?assertEqual(1, tracked_user_connection_count(Config, Username)), + ?assertEqual(5, tracked_user_channel_count(Config, Username)), + ?assertEqual(true, is_process_alive(Conn1)), + [?assertEqual(true, is_process_alive(Ch)) || Ch <- Chans1], + + [Conn2] = open_connections(Config, [{0, Username2}]), + Chans2 = [_|_] = open_channels(Conn2, 5), + ?assertEqual(1, count_connections_in(Config, Username2)), + ?assertEqual(5, count_channels_in(Config, Username2)), + ?assertEqual(1, tracked_user_connection_count(Config, Username2)), + ?assertEqual(5, tracked_user_channel_count(Config, Username2)), + ?assertEqual(true, is_process_alive(Conn2)), + [?assertEqual(true, is_process_alive(Ch)) || Ch <- Chans2], + + ?assertEqual(true, exists_in_tracked_connection_per_vhost_table(Config, Vhost)), + + rabbit_ct_broker_helpers:delete_vhost(Config, Vhost), + timer:sleep(200), + ?assertEqual(0, count_connections_in(Config, Username2)), + ?assertEqual(0, count_channels_in(Config, Username2)), + ?assertEqual(0, tracked_user_connection_count(Config, Username2)), + ?assertEqual(0, tracked_user_channel_count(Config, Username2)), + ?assertEqual(false, is_process_alive(Conn2)), + [?assertEqual(false, is_process_alive(Ch)) || Ch <- Chans2], + + ?assertEqual(0, count_connections_in(Config, Username)), + ?assertEqual(0, count_channels_in(Config, Username)), + ?assertEqual(0, tracked_user_connection_count(Config, Username)), + ?assertEqual(0, tracked_user_channel_count(Config, Username)), + ?assertEqual(false, is_process_alive(Conn1)), + [?assertEqual(false, is_process_alive(Ch)) || Ch <- Chans1], + + %% ensure vhost entry is cleared after 'tracking_execution_timeout' + ?assertEqual(false, exists_in_tracked_connection_per_vhost_table(Config, Vhost)), + + rabbit_ct_broker_helpers:add_vhost(Config, Vhost). + +single_node_vhost_down_mimic(Config) -> + Username = proplists:get_value(rmq_username, Config), + Username2 = <<"guest2">>, + + Vhost = proplists:get_value(rmq_vhost, Config), + + rabbit_ct_broker_helpers:add_user(Config, Username2), + rabbit_ct_broker_helpers:set_full_permissions(Config, Username2, Vhost), + + ?assertEqual(0, count_connections_in(Config, Username)), + ?assertEqual(0, count_connections_in(Config, Username2)), + ?assertEqual(0, count_channels_in(Config, Username)), + ?assertEqual(0, count_channels_in(Config, Username2)), + ?assertEqual(0, tracked_user_connection_count(Config, Username)), + ?assertEqual(0, tracked_user_connection_count(Config, Username2)), + ?assertEqual(0, tracked_user_channel_count(Config, Username)), + ?assertEqual(0, tracked_user_channel_count(Config, Username2)), + + [Conn1] = open_connections(Config, [0]), + Chans1 = [_|_] = open_channels(Conn1, 5), + ?assertEqual(1, count_connections_in(Config, Username)), + ?assertEqual(5, count_channels_in(Config, Username)), + ?assertEqual(1, tracked_user_connection_count(Config, Username)), + ?assertEqual(5, tracked_user_channel_count(Config, Username)), + ?assertEqual(true, is_process_alive(Conn1)), + [?assertEqual(true, is_process_alive(Ch)) || Ch <- Chans1], + + [Conn2] = open_connections(Config, [{0, Username2}]), + Chans2 = [_|_] = open_channels(Conn2, 5), + ?assertEqual(1, count_connections_in(Config, Username2)), + ?assertEqual(5, count_channels_in(Config, Username2)), + ?assertEqual(1, tracked_user_connection_count(Config, Username2)), + ?assertEqual(5, tracked_user_channel_count(Config, Username2)), + ?assertEqual(true, is_process_alive(Conn2)), + [?assertEqual(true, is_process_alive(Ch)) || Ch <- Chans2], + + %% mimic vhost down event, while connections exist + mimic_vhost_down(Config, 0, Vhost), + timer:sleep(200), + ?assertEqual(0, count_connections_in(Config, Username2)), + ?assertEqual(0, count_channels_in(Config, Username2)), + ?assertEqual(0, tracked_user_connection_count(Config, Username2)), + ?assertEqual(0, tracked_user_channel_count(Config, Username2)), + ?assertEqual(false, is_process_alive(Conn2)), + [?assertEqual(false, is_process_alive(Ch)) || Ch <- Chans2], + + ?assertEqual(0, count_connections_in(Config, Username)), + ?assertEqual(0, count_channels_in(Config, Username)), + ?assertEqual(0, tracked_user_connection_count(Config, Username)), + ?assertEqual(0, tracked_user_channel_count(Config, Username)), + ?assertEqual(false, is_process_alive(Conn1)), + [?assertEqual(false, is_process_alive(Ch)) || Ch <- Chans1]. + +cluster_user_deletion(Config) -> + set_tracking_execution_timeout(Config, 0, 100), + set_tracking_execution_timeout(Config, 1, 100), + Username = proplists:get_value(rmq_username, Config), + Username2 = <<"guest2">>, + + Vhost = proplists:get_value(rmq_vhost, Config), + + rabbit_ct_broker_helpers:add_user(Config, Username2), + rabbit_ct_broker_helpers:set_full_permissions(Config, Username2, Vhost), + + ?assertEqual(100, get_tracking_execution_timeout(Config, 0)), + ?assertEqual(100, get_tracking_execution_timeout(Config, 1)), + + ?assertEqual(0, count_connections_in(Config, Username)), + ?assertEqual(0, count_connections_in(Config, Username2)), + ?assertEqual(0, count_channels_in(Config, Username)), + ?assertEqual(0, count_channels_in(Config, Username2)), + ?assertEqual(0, tracked_user_connection_count(Config, Username)), + ?assertEqual(0, tracked_user_connection_count(Config, Username2)), + ?assertEqual(0, tracked_user_channel_count(Config, Username)), + ?assertEqual(0, tracked_user_channel_count(Config, Username2)), + + [Conn1] = open_connections(Config, [0]), + Chans1 = [_|_] = open_channels(Conn1, 5), + ?assertEqual(1, count_connections_in(Config, Username)), + ?assertEqual(5, count_channels_in(Config, Username)), + ?assertEqual(1, tracked_user_connection_count(Config, Username)), + ?assertEqual(5, tracked_user_channel_count(Config, Username)), + ?assertEqual(true, is_process_alive(Conn1)), + [?assertEqual(true, is_process_alive(Ch)) || Ch <- Chans1], + + [Conn2] = open_connections(Config, [{1, Username2}]), + Chans2 = [_|_] = open_channels(Conn2, 5), + ?assertEqual(1, count_connections_in(Config, Username2)), + ?assertEqual(5, count_channels_in(Config, Username2)), + ?assertEqual(1, tracked_user_connection_count(Config, Username2)), + ?assertEqual(5, tracked_user_channel_count(Config, Username2)), + ?assertEqual(true, is_process_alive(Conn2)), + [?assertEqual(true, is_process_alive(Ch)) || Ch <- Chans2], + + ?assertEqual(true, exists_in_tracked_connection_per_user_table(Config, 1, Username2)), + ?assertEqual(true, exists_in_tracked_channel_per_user_table(Config, 1, Username2)), + + rabbit_ct_broker_helpers:delete_user(Config, Username2), + timer:sleep(200), + ?assertEqual(0, count_connections_in(Config, Username2)), + ?assertEqual(0, count_channels_in(Config, Username2)), + ?assertEqual(0, tracked_user_connection_count(Config, Username2)), + ?assertEqual(0, tracked_user_channel_count(Config, Username2)), + ?assertEqual(false, is_process_alive(Conn2)), + [?assertEqual(false, is_process_alive(Ch)) || Ch <- Chans2], + + %% ensure user entry is cleared after 'tracking_execution_timeout' + ?assertEqual(false, exists_in_tracked_connection_per_user_table(Config, 1, Username2)), + ?assertEqual(false, exists_in_tracked_channel_per_user_table(Config, 1, Username2)), + + close_channels(Chans1), + ?assertEqual(0, count_channels_in(Config, Username)), + ?assertEqual(0, tracked_user_channel_count(Config, Username)), + + close_connections([Conn1]), + ?assertEqual(0, count_connections_in(Config, Username)), + ?assertEqual(0, tracked_user_connection_count(Config, Username)). + +cluster_vhost_deletion(Config) -> + set_tracking_execution_timeout(Config, 0, 100), + set_tracking_execution_timeout(Config, 1, 100), + Username = proplists:get_value(rmq_username, Config), + Username2 = <<"guest2">>, + + Vhost = proplists:get_value(rmq_vhost, Config), + + rabbit_ct_broker_helpers:add_user(Config, Username2), + rabbit_ct_broker_helpers:set_full_permissions(Config, Username2, Vhost), + + ?assertEqual(100, get_tracking_execution_timeout(Config, 0)), + ?assertEqual(100, get_tracking_execution_timeout(Config, 1)), + + ?assertEqual(0, count_connections_in(Config, Username)), + ?assertEqual(0, count_connections_in(Config, Username2)), + ?assertEqual(0, count_channels_in(Config, Username)), + ?assertEqual(0, count_channels_in(Config, Username2)), + ?assertEqual(0, tracked_user_connection_count(Config, Username)), + ?assertEqual(0, tracked_user_connection_count(Config, Username2)), + ?assertEqual(0, tracked_user_channel_count(Config, Username)), + ?assertEqual(0, tracked_user_channel_count(Config, Username2)), + + [Conn1] = open_connections(Config, [{0, Username}]), + Chans1 = [_|_] = open_channels(Conn1, 5), + ?assertEqual(1, count_connections_in(Config, Username)), + ?assertEqual(5, count_channels_in(Config, Username)), + ?assertEqual(1, tracked_user_connection_count(Config, Username)), + ?assertEqual(5, tracked_user_channel_count(Config, Username)), + ?assertEqual(true, is_process_alive(Conn1)), + [?assertEqual(true, is_process_alive(Ch)) || Ch <- Chans1], + + [Conn2] = open_connections(Config, [{1, Username2}]), + Chans2 = [_|_] = open_channels(Conn2, 5), + ?assertEqual(1, count_connections_in(Config, Username2)), + ?assertEqual(5, count_channels_in(Config, Username2)), + ?assertEqual(1, tracked_user_connection_count(Config, Username2)), + ?assertEqual(5, tracked_user_channel_count(Config, Username2)), + ?assertEqual(true, is_process_alive(Conn2)), + [?assertEqual(true, is_process_alive(Ch)) || Ch <- Chans2], + + ?assertEqual(true, exists_in_tracked_connection_per_vhost_table(Config, 0, Vhost)), + ?assertEqual(true, exists_in_tracked_connection_per_vhost_table(Config, 1, Vhost)), + + rabbit_ct_broker_helpers:delete_vhost(Config, Vhost), + timer:sleep(200), + ?assertEqual(0, count_connections_in(Config, Username2)), + ?assertEqual(0, count_channels_in(Config, Username2)), + ?assertEqual(0, tracked_user_connection_count(Config, Username2)), + ?assertEqual(0, tracked_user_channel_count(Config, Username2)), + ?assertEqual(false, is_process_alive(Conn2)), + [?assertEqual(false, is_process_alive(Ch)) || Ch <- Chans2], + + ?assertEqual(0, count_connections_in(Config, Username)), + ?assertEqual(0, count_channels_in(Config, Username)), + ?assertEqual(0, tracked_user_connection_count(Config, Username)), + ?assertEqual(0, tracked_user_channel_count(Config, Username)), + ?assertEqual(false, is_process_alive(Conn1)), + [?assertEqual(false, is_process_alive(Ch)) || Ch <- Chans1], + + %% ensure vhost entry is cleared after 'tracking_execution_timeout' + ?assertEqual(false, exists_in_tracked_connection_per_vhost_table(Config, 0, Vhost)), + ?assertEqual(false, exists_in_tracked_connection_per_vhost_table(Config, 1, Vhost)), + + rabbit_ct_broker_helpers:add_vhost(Config, Vhost), + rabbit_ct_broker_helpers:add_user(Config, Username), + rabbit_ct_broker_helpers:set_full_permissions(Config, Username, Vhost). + +cluster_vhost_down_mimic(Config) -> + Username = proplists:get_value(rmq_username, Config), + Username2 = <<"guest2">>, + + Vhost = proplists:get_value(rmq_vhost, Config), + + rabbit_ct_broker_helpers:add_user(Config, Username2), + rabbit_ct_broker_helpers:set_full_permissions(Config, Username2, Vhost), + + ?assertEqual(0, count_connections_in(Config, Username)), + ?assertEqual(0, count_connections_in(Config, Username2)), + ?assertEqual(0, count_channels_in(Config, Username)), + ?assertEqual(0, count_channels_in(Config, Username2)), + ?assertEqual(0, tracked_user_connection_count(Config, Username)), + ?assertEqual(0, tracked_user_connection_count(Config, Username2)), + ?assertEqual(0, tracked_user_channel_count(Config, Username)), + ?assertEqual(0, tracked_user_channel_count(Config, Username2)), + + [Conn1] = open_connections(Config, [{0, Username}]), + Chans1 = [_|_] = open_channels(Conn1, 5), + ?assertEqual(1, count_connections_in(Config, Username)), + ?assertEqual(5, count_channels_in(Config, Username)), + ?assertEqual(1, tracked_user_connection_count(Config, Username)), + ?assertEqual(5, tracked_user_channel_count(Config, Username)), + ?assertEqual(true, is_process_alive(Conn1)), + [?assertEqual(true, is_process_alive(Ch)) || Ch <- Chans1], + + [Conn2] = open_connections(Config, [{1, Username2}]), + Chans2 = [_|_] = open_channels(Conn2, 5), + ?assertEqual(1, count_connections_in(Config, Username2)), + ?assertEqual(5, count_channels_in(Config, Username2)), + ?assertEqual(1, tracked_user_connection_count(Config, Username2)), + ?assertEqual(5, tracked_user_channel_count(Config, Username2)), + ?assertEqual(true, is_process_alive(Conn2)), + [?assertEqual(true, is_process_alive(Ch)) || Ch <- Chans2], + + mimic_vhost_down(Config, 1, Vhost), + timer:sleep(100), + ?assertEqual(0, count_connections_in(Config, Username2)), + ?assertEqual(0, count_channels_in(Config, Username2)), + ?assertEqual(0, tracked_user_connection_count(Config, Username2)), + ?assertEqual(0, tracked_user_channel_count(Config, Username2)), + ?assertEqual(false, is_process_alive(Conn2)), + [?assertEqual(false, is_process_alive(Ch)) || Ch <- Chans2], + + %% gen_event notifies local handlers. remote connections still active + ?assertEqual(1, count_connections_in(Config, Username)), + ?assertEqual(5, count_channels_in(Config, Username)), + ?assertEqual(1, tracked_user_connection_count(Config, Username)), + ?assertEqual(5, tracked_user_channel_count(Config, Username)), + ?assertEqual(true, is_process_alive(Conn1)), + [?assertEqual(true, is_process_alive(Ch)) || Ch <- Chans1], + + mimic_vhost_down(Config, 0, Vhost), + timer:sleep(100), + ?assertEqual(0, count_connections_in(Config, Username)), + ?assertEqual(0, count_channels_in(Config, Username)), + ?assertEqual(0, tracked_user_connection_count(Config, Username)), + ?assertEqual(0, tracked_user_channel_count(Config, Username)), + ?assertEqual(false, is_process_alive(Conn1)), + [?assertEqual(false, is_process_alive(Ch)) || Ch <- Chans1]. + +cluster_node_removed(Config) -> + Username = proplists:get_value(rmq_username, Config), + Username2 = <<"guest2">>, + + Vhost = proplists:get_value(rmq_vhost, Config), + + rabbit_ct_broker_helpers:add_user(Config, Username2), + rabbit_ct_broker_helpers:set_full_permissions(Config, Username2, Vhost), + + ?assertEqual(0, count_connections_in(Config, Username)), + ?assertEqual(0, count_connections_in(Config, Username2)), + ?assertEqual(0, count_channels_in(Config, Username)), + ?assertEqual(0, count_channels_in(Config, Username2)), + ?assertEqual(0, tracked_user_connection_count(Config, Username)), + ?assertEqual(0, tracked_user_connection_count(Config, Username2)), + ?assertEqual(0, tracked_user_channel_count(Config, Username)), + ?assertEqual(0, tracked_user_channel_count(Config, Username2)), + + [Conn1] = open_connections(Config, [{0, Username}]), + Chans1 = [_|_] = open_channels(Conn1, 5), + ?assertEqual(1, count_connections_in(Config, Username)), + ?assertEqual(5, count_channels_in(Config, Username)), + ?assertEqual(1, tracked_user_connection_count(Config, Username)), + ?assertEqual(5, tracked_user_channel_count(Config, Username)), + ?assertEqual(true, is_process_alive(Conn1)), + [?assertEqual(true, is_process_alive(Ch)) || Ch <- Chans1], + + [Conn2] = open_connections(Config, [{1, Username2}]), + Chans2 = [_|_] = open_channels(Conn2, 5), + ?assertEqual(1, count_connections_in(Config, Username2)), + ?assertEqual(5, count_channels_in(Config, Username2)), + ?assertEqual(1, tracked_user_connection_count(Config, Username2)), + ?assertEqual(5, tracked_user_channel_count(Config, Username2)), + ?assertEqual(true, is_process_alive(Conn2)), + [?assertEqual(true, is_process_alive(Ch)) || Ch <- Chans2], + + rabbit_ct_broker_helpers:stop_broker(Config, 1), + timer:sleep(200), + ?assertEqual(1, count_connections_in(Config, Username)), + ?assertEqual(5, count_channels_in(Config, Username)), + ?assertEqual(1, tracked_user_connection_count(Config, Username)), + ?assertEqual(5, tracked_user_channel_count(Config, Username)), + ?assertEqual(true, is_process_alive(Conn1)), + [?assertEqual(true, is_process_alive(Ch)) || Ch <- Chans1], + + rabbit_ct_broker_helpers:forget_cluster_node(Config, 0, 1), + timer:sleep(200), + NodeName = rabbit_ct_broker_helpers:get_node_config(Config, 1, nodename), + + DroppedConnTrackingTables = + rabbit_connection_tracking:get_all_tracked_connection_table_names_for_node(NodeName), + [?assertEqual( + {'EXIT', {aborted, {no_exists, Tab, all}}}, + catch mnesia:table_info(Tab, all)) || Tab <- DroppedConnTrackingTables], + + DroppedChTrackingTables = + rabbit_channel_tracking:get_all_tracked_channel_table_names_for_node(NodeName), + [?assertEqual( + {'EXIT', {aborted, {no_exists, Tab, all}}}, + catch mnesia:table_info(Tab, all)) || Tab <- DroppedChTrackingTables], + + ?assertEqual(false, is_process_alive(Conn2)), + [?assertEqual(false, is_process_alive(Ch)) || Ch <- Chans2], + + ?assertEqual(1, count_connections_in(Config, Username)), + ?assertEqual(5, count_channels_in(Config, Username)), + ?assertEqual(1, tracked_user_connection_count(Config, Username)), + ?assertEqual(5, tracked_user_channel_count(Config, Username)), + ?assertEqual(true, is_process_alive(Conn1)), + [?assertEqual(true, is_process_alive(Ch)) || Ch <- Chans1], + + close_channels(Chans1), + ?assertEqual(0, count_channels_in(Config, Username)), + ?assertEqual(0, tracked_user_channel_count(Config, Username)), + + close_connections([Conn1]), + ?assertEqual(0, count_connections_in(Config, Username)), + ?assertEqual(0, tracked_user_connection_count(Config, Username)). + +%% ------------------------------------------------------------------- +%% Helpers +%% ------------------------------------------------------------------- + +open_connections(Config, NodesAndUsers) -> + % Randomly select connection type + OpenConnectionFun = case ?config(connection_type, Config) of + network -> open_unmanaged_connection; + direct -> open_unmanaged_connection_direct + end, + Conns = lists:map(fun + ({Node, User}) -> + rabbit_ct_client_helpers:OpenConnectionFun(Config, Node, + User, User); + (Node) -> + rabbit_ct_client_helpers:OpenConnectionFun(Config, Node) + end, NodesAndUsers), + timer:sleep(500), + Conns. + +close_connections(Conns) -> + lists:foreach(fun + (Conn) -> + rabbit_ct_client_helpers:close_connection(Conn) + end, Conns), + timer:sleep(500). + +kill_connections(Conns) -> + lists:foreach(fun + (Conn) -> + (catch exit(Conn, please_terminate)) + end, Conns), + timer:sleep(500). + +open_channels(Conn, N) -> + [begin + {ok, Ch} = amqp_connection:open_channel(Conn), + Ch + end || _ <- lists:seq(1, N)]. + +close_channels(Channels = [_|_]) -> + [rabbit_ct_client_helpers:close_channel(Ch) || Ch <- Channels]. + +count_connections_in(Config, Username) -> + length(connections_in(Config, Username)). + +connections_in(Config, Username) -> + connections_in(Config, 0, Username). +connections_in(Config, NodeIndex, Username) -> + tracked_list_of_user(Config, NodeIndex, rabbit_connection_tracking, Username). + +count_channels_in(Config, Username) -> + Channels = channels_in(Config, Username), + length([Ch || Ch = #tracked_channel{username = Username0} <- Channels, + Username =:= Username0]). + +channels_in(Config, Username) -> + channels_in(Config, 0, Username). +channels_in(Config, NodeIndex, Username) -> + tracked_list_of_user(Config, NodeIndex, rabbit_channel_tracking, Username). + +tracked_list_of_user(Config, NodeIndex, TrackingMod, Username) -> + rabbit_ct_broker_helpers:rpc(Config, NodeIndex, + TrackingMod, + list_of_user, [Username]). + +tracked_user_connection_count(Config, Username) -> + tracked_user_connection_count(Config, 0, Username). +tracked_user_connection_count(Config, NodeIndex, Username) -> + count_user_tracked_items(Config, NodeIndex, rabbit_connection_tracking, Username). + +tracked_user_channel_count(Config, Username) -> + tracked_user_channel_count(Config, 0, Username). +tracked_user_channel_count(Config, NodeIndex, Username) -> + count_user_tracked_items(Config, NodeIndex, rabbit_channel_tracking, Username). + +count_user_tracked_items(Config, NodeIndex, TrackingMod, Username) -> + rabbit_ct_broker_helpers:rpc(Config, NodeIndex, + TrackingMod, + count_tracked_items_in, [{user, Username}]). + +exists_in_tracked_connection_per_vhost_table(Config, VHost) -> + exists_in_tracked_connection_per_vhost_table(Config, 0, VHost). +exists_in_tracked_connection_per_vhost_table(Config, NodeIndex, VHost) -> + exists_in_tracking_table(Config, NodeIndex, + fun rabbit_connection_tracking:tracked_connection_per_vhost_table_name_for/1, + VHost). + +exists_in_tracked_connection_per_user_table(Config, Username) -> + exists_in_tracked_connection_per_user_table(Config, 0, Username). +exists_in_tracked_connection_per_user_table(Config, NodeIndex, Username) -> + exists_in_tracking_table(Config, NodeIndex, + fun rabbit_connection_tracking:tracked_connection_per_user_table_name_for/1, + Username). + +exists_in_tracked_channel_per_user_table(Config, Username) -> + exists_in_tracked_channel_per_user_table(Config, 0, Username). +exists_in_tracked_channel_per_user_table(Config, NodeIndex, Username) -> + exists_in_tracking_table(Config, NodeIndex, + fun rabbit_channel_tracking:tracked_channel_per_user_table_name_for/1, + Username). + +exists_in_tracking_table(Config, NodeIndex, TableNameFun, Key) -> + Node = rabbit_ct_broker_helpers:get_node_config( + Config, NodeIndex, nodename), + Tab = TableNameFun(Node), + AllKeys = rabbit_ct_broker_helpers:rpc(Config, NodeIndex, + mnesia, + dirty_all_keys, [Tab]), + lists:member(Key, AllKeys). + +mimic_vhost_down(Config, NodeIndex, VHost) -> + rabbit_ct_broker_helpers:rpc(Config, NodeIndex, + rabbit_vhost, vhost_down, [VHost]). + +all_connections(Config) -> + all_connections(Config, 0). +all_connections(Config, NodeIndex) -> + all_tracked_items(Config, NodeIndex, rabbit_connection_tracking). + +all_channels(Config) -> + all_channels(Config, 0). +all_channels(Config, NodeIndex) -> + all_tracked_items(Config, NodeIndex, rabbit_channel_tracking). + +all_tracked_items(Config, NodeIndex, TrackingMod) -> + rabbit_ct_broker_helpers:rpc(Config, NodeIndex, + TrackingMod, + list, []). + +set_up_vhost(Config, VHost) -> + rabbit_ct_broker_helpers:add_vhost(Config, VHost), + rabbit_ct_broker_helpers:set_full_permissions(Config, <<"guest">>, VHost), + set_vhost_connection_limit(Config, VHost, -1). + +set_vhost_connection_limit(Config, VHost, Count) -> + set_vhost_connection_limit(Config, 0, VHost, Count). + +set_vhost_connection_limit(Config, NodeIndex, VHost, Count) -> + Node = rabbit_ct_broker_helpers:get_node_config( + Config, NodeIndex, nodename), + ok = rabbit_ct_broker_helpers:control_action( + set_vhost_limits, Node, + ["{\"max-connections\": " ++ integer_to_list(Count) ++ "}"], + [{"-p", binary_to_list(VHost)}]). + +set_tracking_execution_timeout(Config, Timeout) -> + set_tracking_execution_timeout(Config, 0, Timeout). +set_tracking_execution_timeout(Config, NodeIndex, Timeout) -> + rabbit_ct_broker_helpers:rpc(Config, NodeIndex, + application, set_env, + [rabbit, tracking_execution_timeout, Timeout]). + +get_tracking_execution_timeout(Config) -> + get_tracking_execution_timeout(Config, 0). +get_tracking_execution_timeout(Config, NodeIndex) -> + {ok, Timeout} = rabbit_ct_broker_helpers:rpc( + Config, NodeIndex, + application, get_env, + [rabbit, tracking_execution_timeout]), + Timeout. + +await_running_node_refresh(_Config, _NodeIndex) -> + timer:sleep(250). + +expect_that_client_connection_is_rejected(Config) -> + expect_that_client_connection_is_rejected(Config, 0). + +expect_that_client_connection_is_rejected(Config, NodeIndex) -> + {error, not_allowed} = + rabbit_ct_client_helpers:open_unmanaged_connection(Config, NodeIndex). + +expect_that_client_connection_is_rejected(Config, NodeIndex, VHost) -> + {error, not_allowed} = + rabbit_ct_client_helpers:open_unmanaged_connection(Config, NodeIndex, VHost). diff --git a/test/per_vhost_connection_limit_SUITE.erl b/test/per_vhost_connection_limit_SUITE.erl index f3e37b0a0c..ef9e5c4554 100644 --- a/test/per_vhost_connection_limit_SUITE.erl +++ b/test/per_vhost_connection_limit_SUITE.erl @@ -137,7 +137,7 @@ clear_all_connection_tracking_tables(Config) -> [rabbit_ct_broker_helpers:rpc(Config, N, rabbit_connection_tracking, - clear_tracked_connection_tables_for_this_node, + clear_tracking_tables, []) || N <- rabbit_ct_broker_helpers:get_node_configs(Config, nodename)]. %% ------------------------------------------------------------------- @@ -692,7 +692,7 @@ count_connections_in(Config, VHost, NodeIndex) -> timer:sleep(200), rabbit_ct_broker_helpers:rpc(Config, NodeIndex, rabbit_connection_tracking, - count_connections_in, [VHost]). + count_tracked_items_in, [{vhost, VHost}]). connections_in(Config, VHost) -> connections_in(Config, 0, VHost). diff --git a/test/per_vhost_connection_limit_partitions_SUITE.erl b/test/per_vhost_connection_limit_partitions_SUITE.erl index e4c6864ea0..2748d95592 100644 --- a/test/per_vhost_connection_limit_partitions_SUITE.erl +++ b/test/per_vhost_connection_limit_partitions_SUITE.erl @@ -140,7 +140,7 @@ count_connections_in(Config, VHost) -> count_connections_in(Config, VHost, NodeIndex) -> rabbit_ct_broker_helpers:rpc(Config, NodeIndex, rabbit_connection_tracking, - count_connections_in, [VHost]). + count_tracked_items_in, [{vhost, VHost}]). connections_in(Config, VHost) -> connections_in(Config, 0, VHost). @@ -148,22 +148,3 @@ connections_in(Config, NodeIndex, VHost) -> rabbit_ct_broker_helpers:rpc(Config, NodeIndex, rabbit_connection_tracking, list, [VHost]). - -connections_on_node(Config) -> - connections_on_node(Config, 0). -connections_on_node(Config, NodeIndex) -> - Node = rabbit_ct_broker_helpers:get_node_config(Config, NodeIndex, nodename), - rabbit_ct_broker_helpers:rpc(Config, NodeIndex, - rabbit_connection_tracking, - list_on_node, [Node]). -connections_on_node(Config, NodeIndex, NodeForListing) -> - rabbit_ct_broker_helpers:rpc(Config, NodeIndex, - rabbit_connection_tracking, - list_on_node, [NodeForListing]). - -all_connections(Config) -> - all_connections(Config, 0). -all_connections(Config, NodeIndex) -> - rabbit_ct_broker_helpers:rpc(Config, NodeIndex, - rabbit_connection_tracking, - list, []). diff --git a/test/unit_access_control_SUITE.erl b/test/unit_access_control_SUITE.erl index fcfd9e2bde..af8f481083 100644 --- a/test/unit_access_control_SUITE.erl +++ b/test/unit_access_control_SUITE.erl @@ -94,23 +94,17 @@ password_hashing1(_Config) -> rabbit_password_hashing_md5 = rabbit_auth_backend_internal:hashing_module_for_user( - #internal_user{}), + internal_user:new()), rabbit_password_hashing_md5 = rabbit_auth_backend_internal:hashing_module_for_user( - #internal_user{ - hashing_algorithm = undefined - }), + internal_user:new({hashing_algorithm, undefined})), rabbit_password_hashing_md5 = rabbit_auth_backend_internal:hashing_module_for_user( - #internal_user{ - hashing_algorithm = rabbit_password_hashing_md5 - }), + internal_user:new({hashing_algorithm, rabbit_password_hashing_md5})), rabbit_password_hashing_sha256 = rabbit_auth_backend_internal:hashing_module_for_user( - #internal_user{ - hashing_algorithm = rabbit_password_hashing_sha256 - }), + internal_user:new({hashing_algorithm, rabbit_password_hashing_sha256})), passed. @@ -211,23 +205,20 @@ set_tags_for_passwordless_user1(_Config) -> ok = rabbit_auth_backend_internal:set_tags(Username, [management], <<"acting-user">>), - ?assertMatch( - {ok, #internal_user{tags = [management]}}, - rabbit_auth_backend_internal:lookup_user(Username)), + {ok, User1} = rabbit_auth_backend_internal:lookup_user(Username), + ?assertEqual([management], internal_user:get_tags(User1)), ok = rabbit_auth_backend_internal:set_tags(Username, [management, policymaker], <<"acting-user">>), - ?assertMatch( - {ok, #internal_user{tags = [management, policymaker]}}, - rabbit_auth_backend_internal:lookup_user(Username)), + {ok, User2} = rabbit_auth_backend_internal:lookup_user(Username), + ?assertEqual([management, policymaker], internal_user:get_tags(User2)), ok = rabbit_auth_backend_internal:set_tags(Username, [], <<"acting-user">>), - ?assertMatch( - {ok, #internal_user{tags = []}}, - rabbit_auth_backend_internal:lookup_user(Username)), + {ok, User3} = rabbit_auth_backend_internal:lookup_user(Username), + ?assertEqual([], internal_user:get_tags(User3)), ok = rabbit_auth_backend_internal:delete_user(Username, <<"acting-user">>), diff --git a/test/vhost_SUITE.erl b/test/vhost_SUITE.erl index 5906dfecb1..4e6ffe0d74 100644 --- a/test/vhost_SUITE.erl +++ b/test/vhost_SUITE.erl @@ -351,7 +351,7 @@ count_connections_in(Config, VHost, NodeIndex) -> timer:sleep(200), rabbit_ct_broker_helpers:rpc(Config, NodeIndex, rabbit_connection_tracking, - count_connections_in, [VHost]). + count_tracked_items_in, [{vhost, VHost}]). set_up_vhost(Config, VHost) -> rabbit_ct_broker_helpers:add_vhost(Config, VHost), |
