diff options
Diffstat (limited to 'deps/rabbit/test')
139 files changed, 42029 insertions, 0 deletions
diff --git a/deps/rabbit/test/amqqueue_backward_compatibility_SUITE.erl b/deps/rabbit/test/amqqueue_backward_compatibility_SUITE.erl new file mode 100644 index 0000000000..a02c4721bc --- /dev/null +++ b/deps/rabbit/test/amqqueue_backward_compatibility_SUITE.erl @@ -0,0 +1,302 @@ +-module(amqqueue_backward_compatibility_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-include("amqqueue.hrl"). + +-export([all/0, + groups/0, + init_per_suite/2, + end_per_suite/2, + init_per_group/2, + end_per_group/2, + init_per_testcase/2, + end_per_testcase/2, + + new_amqqueue_v1_is_amqqueue/1, + new_amqqueue_v2_is_amqqueue/1, + random_term_is_not_amqqueue/1, + + amqqueue_v1_is_durable/1, + amqqueue_v2_is_durable/1, + random_term_is_not_durable/1, + + amqqueue_v1_state_matching/1, + amqqueue_v2_state_matching/1, + random_term_state_matching/1, + + amqqueue_v1_type_matching/1, + amqqueue_v2_type_matching/1, + random_term_type_matching/1, + + upgrade_v1_to_v2/1 + ]). + +-define(long_tuple, {random_tuple, a, b, c, d, e, f, g, h, i, j, k, l, m, + n, o, p, q, r, s, t, u, v, w, x, y, z}). + +all() -> + [ + {group, parallel_tests} + ]. + +groups() -> + [ + {parallel_tests, [parallel], [new_amqqueue_v1_is_amqqueue, + new_amqqueue_v2_is_amqqueue, + random_term_is_not_amqqueue, + amqqueue_v1_is_durable, + amqqueue_v2_is_durable, + random_term_is_not_durable, + amqqueue_v1_state_matching, + amqqueue_v2_state_matching, + random_term_state_matching, + amqqueue_v1_type_matching, + amqqueue_v2_type_matching, + random_term_type_matching]} + ]. + +init_per_suite(_, Config) -> Config. +end_per_suite(_, Config) -> Config. + +init_per_group(_, Config) -> Config. +end_per_group(_, Config) -> Config. + +init_per_testcase(_, Config) -> Config. +end_per_testcase(_, Config) -> Config. + +new_amqqueue_v1_is_amqqueue(_) -> + VHost = <<"/">>, + Name = rabbit_misc:r(VHost, queue, my_amqqueue_v1), + Queue = amqqueue:new_with_version(amqqueue_v1, + Name, + self(), + false, + false, + none, + [], + VHost, + #{}, + ?amqqueue_v1_type), + ?assert(?is_amqqueue(Queue)), + ?assert(?is_amqqueue_v1(Queue)), + ?assert(not ?is_amqqueue_v2(Queue)), + ?assert(?amqqueue_is_classic(Queue)), + ?assert(amqqueue:is_classic(Queue)), + ?assert(not ?amqqueue_is_quorum(Queue)), + ?assert(not ?amqqueue_vhost_equals(Queue, <<"frazzle">>)), + ?assert(?amqqueue_has_valid_pid(Queue)), + ?assert(?amqqueue_pid_equals(Queue, self())), + ?assert(?amqqueue_pids_are_equal(Queue, Queue)), + ?assert(?amqqueue_pid_runs_on_local_node(Queue)), + ?assert(amqqueue:qnode(Queue) == node()). + +new_amqqueue_v2_is_amqqueue(_) -> + VHost = <<"/">>, + Name = rabbit_misc:r(VHost, queue, my_amqqueue_v2), + Queue = amqqueue:new_with_version(amqqueue_v2, + Name, + self(), + false, + false, + none, + [], + VHost, + #{}, + rabbit_classic_queue), + ?assert(?is_amqqueue(Queue)), + ?assert(?is_amqqueue_v2(Queue)), + ?assert(not ?is_amqqueue_v1(Queue)), + ?assert(?amqqueue_is_classic(Queue)), + ?assert(amqqueue:is_classic(Queue)), + ?assert(not ?amqqueue_is_quorum(Queue)), + ?assert(not ?amqqueue_vhost_equals(Queue, <<"frazzle">>)), + ?assert(?amqqueue_has_valid_pid(Queue)), + ?assert(?amqqueue_pid_equals(Queue, self())), + ?assert(?amqqueue_pids_are_equal(Queue, Queue)), + ?assert(?amqqueue_pid_runs_on_local_node(Queue)), + ?assert(amqqueue:qnode(Queue) == node()). + +random_term_is_not_amqqueue(_) -> + Term = ?long_tuple, + ?assert(not ?is_amqqueue(Term)), + ?assert(not ?is_amqqueue_v2(Term)), + ?assert(not ?is_amqqueue_v1(Term)). + +%% ------------------------------------------------------------------- + +amqqueue_v1_is_durable(_) -> + VHost = <<"/">>, + Name = rabbit_misc:r(VHost, queue, my_amqqueue_v1), + TransientQueue = amqqueue:new_with_version(amqqueue_v1, + Name, + self(), + false, + false, + none, + [], + VHost, + #{}, + ?amqqueue_v1_type), + DurableQueue = amqqueue:new_with_version(amqqueue_v1, + Name, + self(), + true, + false, + none, + [], + VHost, + #{}, + ?amqqueue_v1_type), + ?assert(not ?amqqueue_is_durable(TransientQueue)), + ?assert(?amqqueue_is_durable(DurableQueue)). + +amqqueue_v2_is_durable(_) -> + VHost = <<"/">>, + Name = rabbit_misc:r(VHost, queue, my_amqqueue_v1), + TransientQueue = amqqueue:new_with_version(amqqueue_v2, + Name, + self(), + false, + false, + none, + [], + VHost, + #{}, + classic), + DurableQueue = amqqueue:new_with_version(amqqueue_v2, + Name, + self(), + true, + false, + none, + [], + VHost, + #{}, + classic), + ?assert(not ?amqqueue_is_durable(TransientQueue)), + ?assert(?amqqueue_is_durable(DurableQueue)). + +random_term_is_not_durable(_) -> + Term = ?long_tuple, + ?assert(not ?amqqueue_is_durable(Term)). + +%% ------------------------------------------------------------------- + +amqqueue_v1_state_matching(_) -> + VHost = <<"/">>, + Name = rabbit_misc:r(VHost, queue, my_amqqueue_v1), + Queue1 = amqqueue:new_with_version(amqqueue_v1, + Name, + self(), + true, + false, + none, + [], + VHost, + #{}, + ?amqqueue_v1_type), + ?assert(?amqqueue_state_is(Queue1, live)), + Queue2 = amqqueue:set_state(Queue1, stopped), + ?assert(?amqqueue_state_is(Queue2, stopped)). + +amqqueue_v2_state_matching(_) -> + VHost = <<"/">>, + Name = rabbit_misc:r(VHost, queue, my_amqqueue_v1), + Queue1 = amqqueue:new_with_version(amqqueue_v2, + Name, + self(), + true, + false, + none, + [], + VHost, + #{}, + classic), + ?assert(?amqqueue_state_is(Queue1, live)), + Queue2 = amqqueue:set_state(Queue1, stopped), + ?assert(?amqqueue_state_is(Queue2, stopped)). + +random_term_state_matching(_) -> + Term = ?long_tuple, + ?assert(not ?amqqueue_state_is(Term, live)). + +%% ------------------------------------------------------------------- + +amqqueue_v1_type_matching(_) -> + VHost = <<"/">>, + Name = rabbit_misc:r(VHost, queue, my_amqqueue_v1), + Queue = amqqueue:new_with_version(amqqueue_v1, + Name, + self(), + true, + false, + none, + [], + VHost, + #{}, + ?amqqueue_v1_type), + ?assert(?amqqueue_is_classic(Queue)), + ?assert(amqqueue:is_classic(Queue)), + ?assert(not ?amqqueue_is_quorum(Queue)). + +amqqueue_v2_type_matching(_) -> + VHost = <<"/">>, + Name = rabbit_misc:r(VHost, queue, my_amqqueue_v1), + ClassicQueue = amqqueue:new_with_version(amqqueue_v2, + Name, + self(), + true, + false, + none, + [], + VHost, + #{}, + rabbit_classic_queue), + ?assert(?amqqueue_is_classic(ClassicQueue)), + ?assert(amqqueue:is_classic(ClassicQueue)), + ?assert(not ?amqqueue_is_quorum(ClassicQueue)), + ?assert(not amqqueue:is_quorum(ClassicQueue)), + QuorumQueue = amqqueue:new_with_version(amqqueue_v2, + Name, + self(), + true, + false, + none, + [], + VHost, + #{}, + rabbit_quorum_queue), + ?assert(not ?amqqueue_is_classic(QuorumQueue)), + ?assert(not amqqueue:is_classic(QuorumQueue)), + ?assert(?amqqueue_is_quorum(QuorumQueue)), + ?assert(amqqueue:is_quorum(QuorumQueue)). + +random_term_type_matching(_) -> + Term = ?long_tuple, + ?assert(not ?amqqueue_is_classic(Term)), + ?assert(not ?amqqueue_is_quorum(Term)), + ?assertException(error, function_clause, amqqueue:is_classic(Term)), + ?assertException(error, function_clause, amqqueue:is_quorum(Term)). + +%% ------------------------------------------------------------------- + +upgrade_v1_to_v2(_) -> + VHost = <<"/">>, + Name = rabbit_misc:r(VHost, queue, my_amqqueue_v1), + OldQueue = amqqueue:new_with_version(amqqueue_v1, + Name, + self(), + true, + false, + none, + [], + VHost, + #{}, + ?amqqueue_v1_type), + ?assert(?is_amqqueue_v1(OldQueue)), + ?assert(not ?is_amqqueue_v2(OldQueue)), + NewQueue = amqqueue:upgrade_to(amqqueue_v2, OldQueue), + ?assert(not ?is_amqqueue_v1(NewQueue)), + ?assert(?is_amqqueue_v2(NewQueue)). diff --git a/deps/rabbit/test/backing_queue_SUITE.erl b/deps/rabbit/test/backing_queue_SUITE.erl new file mode 100644 index 0000000000..be6004c8b9 --- /dev/null +++ b/deps/rabbit/test/backing_queue_SUITE.erl @@ -0,0 +1,1632 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2011-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(backing_queue_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). +-include("amqqueue.hrl"). + +-compile(export_all). + +-define(PERSISTENT_MSG_STORE, msg_store_persistent). +-define(TRANSIENT_MSG_STORE, msg_store_transient). + +-define(TIMEOUT, 30000). +-define(VHOST, <<"/">>). + +-define(VARIABLE_QUEUE_TESTCASES, [ + variable_queue_dynamic_duration_change, + variable_queue_partial_segments_delta_thing, + variable_queue_all_the_bits_not_covered_elsewhere_A, + variable_queue_all_the_bits_not_covered_elsewhere_B, + variable_queue_drop, + variable_queue_fold_msg_on_disk, + variable_queue_dropfetchwhile, + variable_queue_dropwhile_varying_ram_duration, + variable_queue_fetchwhile_varying_ram_duration, + variable_queue_ack_limiting, + variable_queue_purge, + variable_queue_requeue, + variable_queue_requeue_ram_beta, + variable_queue_fold, + variable_queue_batch_publish, + variable_queue_batch_publish_delivered + ]). + +-define(BACKING_QUEUE_TESTCASES, [ + bq_queue_index, + bq_queue_index_props, + {variable_queue_default, [parallel], ?VARIABLE_QUEUE_TESTCASES}, + {variable_queue_lazy, [parallel], ?VARIABLE_QUEUE_TESTCASES ++ + [variable_queue_mode_change]}, + bq_variable_queue_delete_msg_store_files_callback, + bq_queue_recover + ]). + +all() -> + [ + {group, backing_queue_tests} + ]. + +groups() -> + [ + {backing_queue_tests, [], [ + msg_store, + {backing_queue_embed_limit_0, [], ?BACKING_QUEUE_TESTCASES}, + {backing_queue_embed_limit_1024, [], ?BACKING_QUEUE_TESTCASES} + ]} + ]. + +group(backing_queue_tests) -> + [ + %% Several tests based on lazy queues may take more than 30 minutes. + {timetrap, {hours, 1}} + ]; +group(_) -> + []. + +%% ------------------------------------------------------------------- +%% Testsuite setup/teardown. +%% ------------------------------------------------------------------- + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(Group, Config) -> + case lists:member({group, Group}, all()) of + true -> + ClusterSize = 2, + Config1 = rabbit_ct_helpers:set_config(Config, [ + {rmq_nodename_suffix, Group}, + {rmq_nodes_count, ClusterSize} + ]), + rabbit_ct_helpers:run_steps(Config1, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps() ++ [ + fun(C) -> init_per_group1(Group, C) end, + fun setup_file_handle_cache/1 + ]); + false -> + rabbit_ct_helpers:run_steps(Config, [ + fun(C) -> init_per_group1(Group, C) end + ]) + end. + +init_per_group1(backing_queue_tests, Config) -> + Module = rabbit_ct_broker_helpers:rpc(Config, 0, + application, get_env, [rabbit, backing_queue_module]), + case Module of + {ok, rabbit_priority_queue} -> + rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, setup_backing_queue_test_group, [Config]); + _ -> + {skip, rabbit_misc:format( + "Backing queue module not supported by this test group: ~p~n", + [Module])} + end; +init_per_group1(backing_queue_embed_limit_0, Config) -> + ok = rabbit_ct_broker_helpers:rpc(Config, 0, + application, set_env, [rabbit, queue_index_embed_msgs_below, 0]), + Config; +init_per_group1(backing_queue_embed_limit_1024, Config) -> + ok = rabbit_ct_broker_helpers:rpc(Config, 0, + application, set_env, [rabbit, queue_index_embed_msgs_below, 1024]), + Config; +init_per_group1(variable_queue_default, Config) -> + rabbit_ct_helpers:set_config(Config, {variable_queue_type, default}); +init_per_group1(variable_queue_lazy, Config) -> + rabbit_ct_helpers:set_config(Config, {variable_queue_type, lazy}); +init_per_group1(from_cluster_node1, Config) -> + rabbit_ct_helpers:set_config(Config, {test_direction, {0, 1}}); +init_per_group1(from_cluster_node2, Config) -> + rabbit_ct_helpers:set_config(Config, {test_direction, {1, 0}}); +init_per_group1(_, Config) -> + Config. + +setup_file_handle_cache(Config) -> + ok = rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, setup_file_handle_cache1, []), + Config. + +setup_file_handle_cache1() -> + %% FIXME: Why are we doing this? + application:set_env(rabbit, file_handles_high_watermark, 10), + ok = file_handle_cache:set_limit(10), + ok. + +end_per_group(Group, Config) -> + case lists:member({group, Group}, all()) of + true -> + rabbit_ct_helpers:run_steps(Config, + [fun(C) -> end_per_group1(Group, C) end] ++ + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()); + false -> + Config + end. + +end_per_group1(backing_queue_tests, Config) -> + rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, teardown_backing_queue_test_group, [Config]); +end_per_group1(Group, Config) +when Group =:= backing_queue_embed_limit_0 +orelse Group =:= backing_queue_embed_limit_1024 -> + ok = rabbit_ct_broker_helpers:rpc(Config, 0, + application, set_env, [rabbit, queue_index_embed_msgs_below, + ?config(rmq_queue_index_embed_msgs_below, Config)]), + Config; +end_per_group1(_, Config) -> + Config. + +init_per_testcase(Testcase, Config) when Testcase == variable_queue_requeue; + Testcase == variable_queue_fold -> + ok = rabbit_ct_broker_helpers:rpc( + Config, 0, application, set_env, + [rabbit, queue_explicit_gc_run_operation_threshold, 0]), + rabbit_ct_helpers:testcase_started(Config, Testcase); +init_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_started(Config, Testcase). + +end_per_testcase(Testcase, Config) when Testcase == variable_queue_requeue; + Testcase == variable_queue_fold -> + ok = rabbit_ct_broker_helpers:rpc( + Config, 0, application, set_env, + [rabbit, queue_explicit_gc_run_operation_threshold, 1000]), + rabbit_ct_helpers:testcase_finished(Config, Testcase); +end_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_finished(Config, Testcase). + +%% ------------------------------------------------------------------- +%% Message store. +%% ------------------------------------------------------------------- + +msg_store(Config) -> + passed = rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, msg_store1, [Config]). + +msg_store1(_Config) -> + restart_msg_store_empty(), + MsgIds = [msg_id_bin(M) || M <- lists:seq(1,100)], + {MsgIds1stHalf, MsgIds2ndHalf} = lists:split(length(MsgIds) div 2, MsgIds), + Ref = rabbit_guid:gen(), + {Cap, MSCState} = msg_store_client_init_capture( + ?PERSISTENT_MSG_STORE, Ref), + Ref2 = rabbit_guid:gen(), + {Cap2, MSC2State} = msg_store_client_init_capture( + ?PERSISTENT_MSG_STORE, Ref2), + %% check we don't contain any of the msgs we're about to publish + false = msg_store_contains(false, MsgIds, MSCState), + %% test confirm logic + passed = test_msg_store_confirms([hd(MsgIds)], Cap, MSCState), + %% check we don't contain any of the msgs we're about to publish + false = msg_store_contains(false, MsgIds, MSCState), + %% publish the first half + ok = msg_store_write(MsgIds1stHalf, MSCState), + %% sync on the first half + ok = on_disk_await(Cap, MsgIds1stHalf), + %% publish the second half + ok = msg_store_write(MsgIds2ndHalf, MSCState), + %% check they're all in there + true = msg_store_contains(true, MsgIds, MSCState), + %% publish the latter half twice so we hit the caching and ref + %% count code. We need to do this through a 2nd client since a + %% single client is not supposed to write the same message more + %% than once without first removing it. + ok = msg_store_write(MsgIds2ndHalf, MSC2State), + %% check they're still all in there + true = msg_store_contains(true, MsgIds, MSCState), + %% sync on the 2nd half + ok = on_disk_await(Cap2, MsgIds2ndHalf), + %% cleanup + ok = on_disk_stop(Cap2), + ok = rabbit_msg_store:client_delete_and_terminate(MSC2State), + ok = on_disk_stop(Cap), + %% read them all + MSCState1 = msg_store_read(MsgIds, MSCState), + %% read them all again - this will hit the cache, not disk + MSCState2 = msg_store_read(MsgIds, MSCState1), + %% remove them all + ok = msg_store_remove(MsgIds, MSCState2), + %% check first half doesn't exist + false = msg_store_contains(false, MsgIds1stHalf, MSCState2), + %% check second half does exist + true = msg_store_contains(true, MsgIds2ndHalf, MSCState2), + %% read the second half again + MSCState3 = msg_store_read(MsgIds2ndHalf, MSCState2), + %% read the second half again, just for fun (aka code coverage) + MSCState4 = msg_store_read(MsgIds2ndHalf, MSCState3), + ok = rabbit_msg_store:client_terminate(MSCState4), + %% stop and restart, preserving every other msg in 2nd half + ok = rabbit_variable_queue:stop_msg_store(?VHOST), + ok = rabbit_variable_queue:start_msg_store(?VHOST, + [], {fun ([]) -> finished; + ([MsgId|MsgIdsTail]) + when length(MsgIdsTail) rem 2 == 0 -> + {MsgId, 1, MsgIdsTail}; + ([MsgId|MsgIdsTail]) -> + {MsgId, 0, MsgIdsTail} + end, MsgIds2ndHalf}), + MSCState5 = msg_store_client_init(?PERSISTENT_MSG_STORE, Ref), + %% check we have the right msgs left + lists:foldl( + fun (MsgId, Bool) -> + not(Bool = rabbit_msg_store:contains(MsgId, MSCState5)) + end, false, MsgIds2ndHalf), + ok = rabbit_msg_store:client_terminate(MSCState5), + %% restart empty + restart_msg_store_empty(), + MSCState6 = msg_store_client_init(?PERSISTENT_MSG_STORE, Ref), + %% check we don't contain any of the msgs + false = msg_store_contains(false, MsgIds, MSCState6), + %% publish the first half again + ok = msg_store_write(MsgIds1stHalf, MSCState6), + %% this should force some sort of sync internally otherwise misread + ok = rabbit_msg_store:client_terminate( + msg_store_read(MsgIds1stHalf, MSCState6)), + MSCState7 = msg_store_client_init(?PERSISTENT_MSG_STORE, Ref), + ok = msg_store_remove(MsgIds1stHalf, MSCState7), + ok = rabbit_msg_store:client_terminate(MSCState7), + %% restart empty + restart_msg_store_empty(), %% now safe to reuse msg_ids + %% push a lot of msgs in... at least 100 files worth + {ok, FileSize} = application:get_env(rabbit, msg_store_file_size_limit), + PayloadSizeBits = 65536, + BigCount = trunc(100 * FileSize / (PayloadSizeBits div 8)), + MsgIdsBig = [msg_id_bin(X) || X <- lists:seq(1, BigCount)], + Payload = << 0:PayloadSizeBits >>, + ok = with_msg_store_client( + ?PERSISTENT_MSG_STORE, Ref, + fun (MSCStateM) -> + [ok = rabbit_msg_store:write(MsgId, Payload, MSCStateM) || + MsgId <- MsgIdsBig], + MSCStateM + end), + %% now read them to ensure we hit the fast client-side reading + ok = foreach_with_msg_store_client( + ?PERSISTENT_MSG_STORE, Ref, + fun (MsgId, MSCStateM) -> + {{ok, Payload}, MSCStateN} = rabbit_msg_store:read( + MsgId, MSCStateM), + MSCStateN + end, MsgIdsBig), + %% .., then 3s by 1... + ok = msg_store_remove(?PERSISTENT_MSG_STORE, Ref, + [msg_id_bin(X) || X <- lists:seq(BigCount, 1, -3)]), + %% .., then remove 3s by 2, from the young end first. This hits + %% GC (under 50% good data left, but no empty files. Must GC). + ok = msg_store_remove(?PERSISTENT_MSG_STORE, Ref, + [msg_id_bin(X) || X <- lists:seq(BigCount-1, 1, -3)]), + %% .., then remove 3s by 3, from the young end first. This hits + %% GC... + ok = msg_store_remove(?PERSISTENT_MSG_STORE, Ref, + [msg_id_bin(X) || X <- lists:seq(BigCount-2, 1, -3)]), + %% ensure empty + ok = with_msg_store_client( + ?PERSISTENT_MSG_STORE, Ref, + fun (MSCStateM) -> + false = msg_store_contains(false, MsgIdsBig, MSCStateM), + MSCStateM + end), + %% + passed = test_msg_store_client_delete_and_terminate(), + %% restart empty + restart_msg_store_empty(), + passed. + +restart_msg_store_empty() -> + ok = rabbit_variable_queue:stop_msg_store(?VHOST), + ok = rabbit_variable_queue:start_msg_store(?VHOST, + undefined, {fun (ok) -> finished end, ok}). + +msg_id_bin(X) -> + erlang:md5(term_to_binary(X)). + +on_disk_capture() -> + receive + {await, MsgIds, Pid} -> on_disk_capture([], MsgIds, Pid); + stop -> done + end. + +on_disk_capture([_|_], _Awaiting, Pid) -> + Pid ! {self(), surplus}; +on_disk_capture(OnDisk, Awaiting, Pid) -> + receive + {on_disk, MsgIdsS} -> + MsgIds = gb_sets:to_list(MsgIdsS), + on_disk_capture(OnDisk ++ (MsgIds -- Awaiting), Awaiting -- MsgIds, + Pid); + stop -> + done + after (case Awaiting of [] -> 200; _ -> ?TIMEOUT end) -> + case Awaiting of + [] -> Pid ! {self(), arrived}, on_disk_capture(); + _ -> Pid ! {self(), timeout} + end + end. + +on_disk_await(Pid, MsgIds) when is_list(MsgIds) -> + Pid ! {await, MsgIds, self()}, + receive + {Pid, arrived} -> ok; + {Pid, Error} -> Error + end. + +on_disk_stop(Pid) -> + MRef = erlang:monitor(process, Pid), + Pid ! stop, + receive {'DOWN', MRef, process, Pid, _Reason} -> + ok + end. + +msg_store_client_init_capture(MsgStore, Ref) -> + Pid = spawn(fun on_disk_capture/0), + {Pid, rabbit_vhost_msg_store:client_init(?VHOST, MsgStore, Ref, + fun (MsgIds, _ActionTaken) -> + Pid ! {on_disk, MsgIds} + end, undefined)}. + +msg_store_contains(Atom, MsgIds, MSCState) -> + Atom = lists:foldl( + fun (MsgId, Atom1) when Atom1 =:= Atom -> + rabbit_msg_store:contains(MsgId, MSCState) end, + Atom, MsgIds). + +msg_store_read(MsgIds, MSCState) -> + lists:foldl(fun (MsgId, MSCStateM) -> + {{ok, MsgId}, MSCStateN} = rabbit_msg_store:read( + MsgId, MSCStateM), + MSCStateN + end, MSCState, MsgIds). + +msg_store_write(MsgIds, MSCState) -> + ok = lists:foldl(fun (MsgId, ok) -> + rabbit_msg_store:write(MsgId, MsgId, MSCState) + end, ok, MsgIds). + +msg_store_write_flow(MsgIds, MSCState) -> + ok = lists:foldl(fun (MsgId, ok) -> + rabbit_msg_store:write_flow(MsgId, MsgId, MSCState) + end, ok, MsgIds). + +msg_store_remove(MsgIds, MSCState) -> + rabbit_msg_store:remove(MsgIds, MSCState). + +msg_store_remove(MsgStore, Ref, MsgIds) -> + with_msg_store_client(MsgStore, Ref, + fun (MSCStateM) -> + ok = msg_store_remove(MsgIds, MSCStateM), + MSCStateM + end). + +with_msg_store_client(MsgStore, Ref, Fun) -> + rabbit_msg_store:client_terminate( + Fun(msg_store_client_init(MsgStore, Ref))). + +foreach_with_msg_store_client(MsgStore, Ref, Fun, L) -> + rabbit_msg_store:client_terminate( + lists:foldl(fun (MsgId, MSCState) -> Fun(MsgId, MSCState) end, + msg_store_client_init(MsgStore, Ref), L)). + +test_msg_store_confirms(MsgIds, Cap, MSCState) -> + %% write -> confirmed + ok = msg_store_write(MsgIds, MSCState), + ok = on_disk_await(Cap, MsgIds), + %% remove -> _ + ok = msg_store_remove(MsgIds, MSCState), + ok = on_disk_await(Cap, []), + %% write, remove -> confirmed + ok = msg_store_write(MsgIds, MSCState), + ok = msg_store_remove(MsgIds, MSCState), + ok = on_disk_await(Cap, MsgIds), + %% write, remove, write -> confirmed, confirmed + ok = msg_store_write(MsgIds, MSCState), + ok = msg_store_remove(MsgIds, MSCState), + ok = msg_store_write(MsgIds, MSCState), + ok = on_disk_await(Cap, MsgIds ++ MsgIds), + %% remove, write -> confirmed + ok = msg_store_remove(MsgIds, MSCState), + ok = msg_store_write(MsgIds, MSCState), + ok = on_disk_await(Cap, MsgIds), + %% remove, write, remove -> confirmed + ok = msg_store_remove(MsgIds, MSCState), + ok = msg_store_write(MsgIds, MSCState), + ok = msg_store_remove(MsgIds, MSCState), + ok = on_disk_await(Cap, MsgIds), + %% confirmation on timer-based sync + passed = test_msg_store_confirm_timer(), + passed. + +test_msg_store_confirm_timer() -> + Ref = rabbit_guid:gen(), + MsgId = msg_id_bin(1), + Self = self(), + MSCState = rabbit_vhost_msg_store:client_init( + ?VHOST, + ?PERSISTENT_MSG_STORE, + Ref, + fun (MsgIds, _ActionTaken) -> + case gb_sets:is_member(MsgId, MsgIds) of + true -> Self ! on_disk; + false -> ok + end + end, undefined), + ok = msg_store_write([MsgId], MSCState), + ok = msg_store_keep_busy_until_confirm([msg_id_bin(2)], MSCState, false), + ok = msg_store_remove([MsgId], MSCState), + ok = rabbit_msg_store:client_delete_and_terminate(MSCState), + passed. + +msg_store_keep_busy_until_confirm(MsgIds, MSCState, Blocked) -> + After = case Blocked of + false -> 0; + true -> ?MAX_WAIT + end, + Recurse = fun () -> msg_store_keep_busy_until_confirm( + MsgIds, MSCState, credit_flow:blocked()) end, + receive + on_disk -> ok; + {bump_credit, Msg} -> credit_flow:handle_bump_msg(Msg), + Recurse() + after After -> + ok = msg_store_write_flow(MsgIds, MSCState), + ok = msg_store_remove(MsgIds, MSCState), + Recurse() + end. + +test_msg_store_client_delete_and_terminate() -> + restart_msg_store_empty(), + MsgIds = [msg_id_bin(M) || M <- lists:seq(1, 10)], + Ref = rabbit_guid:gen(), + MSCState = msg_store_client_init(?PERSISTENT_MSG_STORE, Ref), + ok = msg_store_write(MsgIds, MSCState), + %% test the 'dying client' fast path for writes + ok = rabbit_msg_store:client_delete_and_terminate(MSCState), + passed. + +%% ------------------------------------------------------------------- +%% Backing queue. +%% ------------------------------------------------------------------- + +setup_backing_queue_test_group(Config) -> + {ok, FileSizeLimit} = + application:get_env(rabbit, msg_store_file_size_limit), + application:set_env(rabbit, msg_store_file_size_limit, 512), + {ok, MaxJournal} = + application:get_env(rabbit, queue_index_max_journal_entries), + application:set_env(rabbit, queue_index_max_journal_entries, 128), + application:set_env(rabbit, msg_store_file_size_limit, + FileSizeLimit), + {ok, Bytes} = + application:get_env(rabbit, queue_index_embed_msgs_below), + rabbit_ct_helpers:set_config(Config, [ + {rmq_queue_index_max_journal_entries, MaxJournal}, + {rmq_queue_index_embed_msgs_below, Bytes} + ]). + +teardown_backing_queue_test_group(Config) -> + %% FIXME: Undo all the setup function did. + application:set_env(rabbit, queue_index_max_journal_entries, + ?config(rmq_queue_index_max_journal_entries, Config)), + %% We will have restarted the message store, and thus changed + %% the order of the children of rabbit_sup. This will cause + %% problems if there are subsequent failures - see bug 24262. + ok = restart_app(), + Config. + +bq_queue_index(Config) -> + passed = rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, bq_queue_index1, [Config]). + +bq_queue_index1(_Config) -> + SegmentSize = rabbit_queue_index:next_segment_boundary(0), + TwoSegs = SegmentSize + SegmentSize, + MostOfASegment = trunc(SegmentSize*0.75), + SeqIdsA = lists:seq(0, MostOfASegment-1), + SeqIdsB = lists:seq(MostOfASegment, 2*MostOfASegment), + SeqIdsC = lists:seq(0, trunc(SegmentSize/2)), + SeqIdsD = lists:seq(0, SegmentSize*4), + + with_empty_test_queue( + fun (Qi0, QName) -> + {0, 0, Qi1} = rabbit_queue_index:bounds(Qi0), + {Qi2, SeqIdsMsgIdsA} = queue_index_publish(SeqIdsA, false, Qi1), + {0, SegmentSize, Qi3} = rabbit_queue_index:bounds(Qi2), + {ReadA, Qi4} = rabbit_queue_index:read(0, SegmentSize, Qi3), + ok = verify_read_with_published(false, false, ReadA, + lists:reverse(SeqIdsMsgIdsA)), + %% should get length back as 0, as all the msgs were transient + {0, 0, Qi6} = restart_test_queue(Qi4, QName), + {0, 0, Qi7} = rabbit_queue_index:bounds(Qi6), + {Qi8, SeqIdsMsgIdsB} = queue_index_publish(SeqIdsB, true, Qi7), + {0, TwoSegs, Qi9} = rabbit_queue_index:bounds(Qi8), + {ReadB, Qi10} = rabbit_queue_index:read(0, SegmentSize, Qi9), + ok = verify_read_with_published(false, true, ReadB, + lists:reverse(SeqIdsMsgIdsB)), + %% should get length back as MostOfASegment + LenB = length(SeqIdsB), + BytesB = LenB * 10, + {LenB, BytesB, Qi12} = restart_test_queue(Qi10, QName), + {0, TwoSegs, Qi13} = rabbit_queue_index:bounds(Qi12), + Qi14 = rabbit_queue_index:deliver(SeqIdsB, Qi13), + {ReadC, Qi15} = rabbit_queue_index:read(0, SegmentSize, Qi14), + ok = verify_read_with_published(true, true, ReadC, + lists:reverse(SeqIdsMsgIdsB)), + Qi16 = rabbit_queue_index:ack(SeqIdsB, Qi15), + Qi17 = rabbit_queue_index:flush(Qi16), + %% Everything will have gone now because #pubs == #acks + {0, 0, Qi18} = rabbit_queue_index:bounds(Qi17), + %% should get length back as 0 because all persistent + %% msgs have been acked + {0, 0, Qi19} = restart_test_queue(Qi18, QName), + Qi19 + end), + + %% These next bits are just to hit the auto deletion of segment files. + %% First, partials: + %% a) partial pub+del+ack, then move to new segment + with_empty_test_queue( + fun (Qi0, _QName) -> + {Qi1, _SeqIdsMsgIdsC} = queue_index_publish(SeqIdsC, + false, Qi0), + Qi2 = rabbit_queue_index:deliver(SeqIdsC, Qi1), + Qi3 = rabbit_queue_index:ack(SeqIdsC, Qi2), + Qi4 = rabbit_queue_index:flush(Qi3), + {Qi5, _SeqIdsMsgIdsC1} = queue_index_publish([SegmentSize], + false, Qi4), + Qi5 + end), + + %% b) partial pub+del, then move to new segment, then ack all in old segment + with_empty_test_queue( + fun (Qi0, _QName) -> + {Qi1, _SeqIdsMsgIdsC2} = queue_index_publish(SeqIdsC, + false, Qi0), + Qi2 = rabbit_queue_index:deliver(SeqIdsC, Qi1), + {Qi3, _SeqIdsMsgIdsC3} = queue_index_publish([SegmentSize], + false, Qi2), + Qi4 = rabbit_queue_index:ack(SeqIdsC, Qi3), + rabbit_queue_index:flush(Qi4) + end), + + %% c) just fill up several segments of all pubs, then +dels, then +acks + with_empty_test_queue( + fun (Qi0, _QName) -> + {Qi1, _SeqIdsMsgIdsD} = queue_index_publish(SeqIdsD, + false, Qi0), + Qi2 = rabbit_queue_index:deliver(SeqIdsD, Qi1), + Qi3 = rabbit_queue_index:ack(SeqIdsD, Qi2), + rabbit_queue_index:flush(Qi3) + end), + + %% d) get messages in all states to a segment, then flush, then do + %% the same again, don't flush and read. This will hit all + %% possibilities in combining the segment with the journal. + with_empty_test_queue( + fun (Qi0, _QName) -> + {Qi1, [Seven,Five,Four|_]} = queue_index_publish([0,1,2,4,5,7], + false, Qi0), + Qi2 = rabbit_queue_index:deliver([0,1,4], Qi1), + Qi3 = rabbit_queue_index:ack([0], Qi2), + Qi4 = rabbit_queue_index:flush(Qi3), + {Qi5, [Eight,Six|_]} = queue_index_publish([3,6,8], false, Qi4), + Qi6 = rabbit_queue_index:deliver([2,3,5,6], Qi5), + Qi7 = rabbit_queue_index:ack([1,2,3], Qi6), + {[], Qi8} = rabbit_queue_index:read(0, 4, Qi7), + {ReadD, Qi9} = rabbit_queue_index:read(4, 7, Qi8), + ok = verify_read_with_published(true, false, ReadD, + [Four, Five, Six]), + {ReadE, Qi10} = rabbit_queue_index:read(7, 9, Qi9), + ok = verify_read_with_published(false, false, ReadE, + [Seven, Eight]), + Qi10 + end), + + %% e) as for (d), but use terminate instead of read, which will + %% exercise journal_minus_segment, not segment_plus_journal. + with_empty_test_queue( + fun (Qi0, QName) -> + {Qi1, _SeqIdsMsgIdsE} = queue_index_publish([0,1,2,4,5,7], + true, Qi0), + Qi2 = rabbit_queue_index:deliver([0,1,4], Qi1), + Qi3 = rabbit_queue_index:ack([0], Qi2), + {5, 50, Qi4} = restart_test_queue(Qi3, QName), + {Qi5, _SeqIdsMsgIdsF} = queue_index_publish([3,6,8], true, Qi4), + Qi6 = rabbit_queue_index:deliver([2,3,5,6], Qi5), + Qi7 = rabbit_queue_index:ack([1,2,3], Qi6), + {5, 50, Qi8} = restart_test_queue(Qi7, QName), + Qi8 + end), + + ok = rabbit_variable_queue:stop(?VHOST), + {ok, _} = rabbit_variable_queue:start(?VHOST, []), + + passed. + +bq_queue_index_props(Config) -> + passed = rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, bq_queue_index_props1, [Config]). + +bq_queue_index_props1(_Config) -> + with_empty_test_queue( + fun(Qi0, _QName) -> + MsgId = rabbit_guid:gen(), + Props = #message_properties{expiry=12345, size = 10}, + Qi1 = rabbit_queue_index:publish( + MsgId, 1, Props, true, infinity, Qi0), + {[{MsgId, 1, Props, _, _}], Qi2} = + rabbit_queue_index:read(1, 2, Qi1), + Qi2 + end), + + ok = rabbit_variable_queue:stop(?VHOST), + {ok, _} = rabbit_variable_queue:start(?VHOST, []), + + passed. + +bq_variable_queue_delete_msg_store_files_callback(Config) -> + passed = rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, bq_variable_queue_delete_msg_store_files_callback1, [Config]). + +bq_variable_queue_delete_msg_store_files_callback1(Config) -> + ok = restart_msg_store_empty(), + QName0 = queue_name(Config, <<"bq_variable_queue_delete_msg_store_files_callback-q">>), + {new, Q} = rabbit_amqqueue:declare(QName0, true, false, [], none, <<"acting-user">>), + QName = amqqueue:get_name(Q), + QPid = amqqueue:get_pid(Q), + Payload = <<0:8388608>>, %% 1MB + Count = 30, + QTState = publish_and_confirm(Q, Payload, Count), + + rabbit_amqqueue:set_ram_duration_target(QPid, 0), + + {ok, Limiter} = rabbit_limiter:start_link(no_id), + + CountMinusOne = Count - 1, + {ok, CountMinusOne, {QName, QPid, _AckTag, false, _Msg}, _} = + rabbit_amqqueue:basic_get(Q, true, Limiter, + <<"bq_variable_queue_delete_msg_store_files_callback1">>, + QTState), + {ok, CountMinusOne} = rabbit_amqqueue:purge(Q), + + %% give the queue a second to receive the close_fds callback msg + timer:sleep(1000), + + rabbit_amqqueue:delete(Q, false, false, <<"acting-user">>), + passed. + +bq_queue_recover(Config) -> + passed = rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, bq_queue_recover1, [Config]). + +bq_queue_recover1(Config) -> + Count = 2 * rabbit_queue_index:next_segment_boundary(0), + QName0 = queue_name(Config, <<"bq_queue_recover-q">>), + {new, Q} = rabbit_amqqueue:declare(QName0, true, false, [], none, <<"acting-user">>), + QName = amqqueue:get_name(Q), + QPid = amqqueue:get_pid(Q), + QT = publish_and_confirm(Q, <<>>, Count), + SupPid = get_queue_sup_pid(Q), + true = is_pid(SupPid), + exit(SupPid, kill), + exit(QPid, kill), + MRef = erlang:monitor(process, QPid), + receive {'DOWN', MRef, process, QPid, _Info} -> ok + after 10000 -> exit(timeout_waiting_for_queue_death) + end, + rabbit_amqqueue:stop(?VHOST), + {Recovered, []} = rabbit_amqqueue:recover(?VHOST), + rabbit_amqqueue:start(Recovered), + {ok, Limiter} = rabbit_limiter:start_link(no_id), + rabbit_amqqueue:with_or_die( + QName, + fun (Q1) when ?is_amqqueue(Q1) -> + QPid1 = amqqueue:get_pid(Q1), + CountMinusOne = Count - 1, + {ok, CountMinusOne, {QName, QPid1, _AckTag, true, _Msg}, _} = + rabbit_amqqueue:basic_get(Q1, false, Limiter, + <<"bq_queue_recover1">>, QT), + exit(QPid1, shutdown), + VQ1 = variable_queue_init(Q, true), + {{_Msg1, true, _AckTag1}, VQ2} = + rabbit_variable_queue:fetch(true, VQ1), + CountMinusOne = rabbit_variable_queue:len(VQ2), + _VQ3 = rabbit_variable_queue:delete_and_terminate(shutdown, VQ2), + ok = rabbit_amqqueue:internal_delete(QName, <<"acting-user">>) + end), + passed. + +%% Return the PID of the given queue's supervisor. +get_queue_sup_pid(Q) when ?is_amqqueue(Q) -> + QName = amqqueue:get_name(Q), + QPid = amqqueue:get_pid(Q), + VHost = QName#resource.virtual_host, + {ok, AmqSup} = rabbit_amqqueue_sup_sup:find_for_vhost(VHost, node(QPid)), + Sups = supervisor:which_children(AmqSup), + get_queue_sup_pid(Sups, QPid). + +get_queue_sup_pid([{_, SupPid, _, _} | Rest], QueuePid) -> + WorkerPids = [Pid || {_, Pid, _, _} <- supervisor:which_children(SupPid)], + case lists:member(QueuePid, WorkerPids) of + true -> SupPid; + false -> get_queue_sup_pid(Rest, QueuePid) + end; +get_queue_sup_pid([], _QueuePid) -> + undefined. + +variable_queue_dynamic_duration_change(Config) -> + passed = rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, variable_queue_dynamic_duration_change1, [Config]). + +variable_queue_dynamic_duration_change1(Config) -> + with_fresh_variable_queue( + fun variable_queue_dynamic_duration_change2/2, + ?config(variable_queue_type, Config)). + +variable_queue_dynamic_duration_change2(VQ0, _QName) -> + SegmentSize = rabbit_queue_index:next_segment_boundary(0), + + %% start by sending in a couple of segments worth + Len = 2*SegmentSize, + VQ1 = variable_queue_publish(false, Len, VQ0), + %% squeeze and relax queue + Churn = Len div 32, + VQ2 = publish_fetch_and_ack(Churn, Len, VQ1), + + {Duration, VQ3} = rabbit_variable_queue:ram_duration(VQ2), + VQ7 = lists:foldl( + fun (Duration1, VQ4) -> + {_Duration, VQ5} = rabbit_variable_queue:ram_duration(VQ4), + VQ6 = variable_queue_set_ram_duration_target( + Duration1, VQ5), + publish_fetch_and_ack(Churn, Len, VQ6) + end, VQ3, [Duration / 4, 0, Duration / 4, infinity]), + + %% drain + {VQ8, AckTags} = variable_queue_fetch(Len, false, false, Len, VQ7), + {_Guids, VQ9} = rabbit_variable_queue:ack(AckTags, VQ8), + {empty, VQ10} = rabbit_variable_queue:fetch(true, VQ9), + + VQ10. + +variable_queue_partial_segments_delta_thing(Config) -> + passed = rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, variable_queue_partial_segments_delta_thing1, [Config]). + +variable_queue_partial_segments_delta_thing1(Config) -> + with_fresh_variable_queue( + fun variable_queue_partial_segments_delta_thing2/2, + ?config(variable_queue_type, Config)). + +variable_queue_partial_segments_delta_thing2(VQ0, _QName) -> + SegmentSize = rabbit_queue_index:next_segment_boundary(0), + HalfSegment = SegmentSize div 2, + OneAndAHalfSegment = SegmentSize + HalfSegment, + VQ1 = variable_queue_publish(true, OneAndAHalfSegment, VQ0), + {_Duration, VQ2} = rabbit_variable_queue:ram_duration(VQ1), + VQ3 = check_variable_queue_status( + variable_queue_set_ram_duration_target(0, VQ2), + %% one segment in q3, and half a segment in delta + [{delta, {delta, SegmentSize, HalfSegment, OneAndAHalfSegment}}, + {q3, SegmentSize}, + {len, SegmentSize + HalfSegment}]), + VQ4 = variable_queue_set_ram_duration_target(infinity, VQ3), + VQ5 = check_variable_queue_status( + variable_queue_publish(true, 1, VQ4), + %% one alpha, but it's in the same segment as the deltas + [{q1, 1}, + {delta, {delta, SegmentSize, HalfSegment, OneAndAHalfSegment}}, + {q3, SegmentSize}, + {len, SegmentSize + HalfSegment + 1}]), + {VQ6, AckTags} = variable_queue_fetch(SegmentSize, true, false, + SegmentSize + HalfSegment + 1, VQ5), + VQ7 = check_variable_queue_status( + VQ6, + %% the half segment should now be in q3 + [{q1, 1}, + {delta, {delta, undefined, 0, undefined}}, + {q3, HalfSegment}, + {len, HalfSegment + 1}]), + {VQ8, AckTags1} = variable_queue_fetch(HalfSegment + 1, true, false, + HalfSegment + 1, VQ7), + {_Guids, VQ9} = rabbit_variable_queue:ack(AckTags ++ AckTags1, VQ8), + %% should be empty now + {empty, VQ10} = rabbit_variable_queue:fetch(true, VQ9), + VQ10. + +variable_queue_all_the_bits_not_covered_elsewhere_A(Config) -> + passed = rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, variable_queue_all_the_bits_not_covered_elsewhere_A1, [Config]). + +variable_queue_all_the_bits_not_covered_elsewhere_A1(Config) -> + with_fresh_variable_queue( + fun variable_queue_all_the_bits_not_covered_elsewhere_A2/2, + ?config(variable_queue_type, Config)). + +variable_queue_all_the_bits_not_covered_elsewhere_A2(VQ0, QName) -> + Count = 2 * rabbit_queue_index:next_segment_boundary(0), + VQ1 = variable_queue_publish(true, Count, VQ0), + VQ2 = variable_queue_publish(false, Count, VQ1), + VQ3 = variable_queue_set_ram_duration_target(0, VQ2), + {VQ4, _AckTags} = variable_queue_fetch(Count, true, false, + Count + Count, VQ3), + {VQ5, _AckTags1} = variable_queue_fetch(Count, false, false, + Count, VQ4), + _VQ6 = rabbit_variable_queue:terminate(shutdown, VQ5), + VQ7 = variable_queue_init(test_amqqueue(QName, true), true), + {{_Msg1, true, _AckTag1}, VQ8} = rabbit_variable_queue:fetch(true, VQ7), + Count1 = rabbit_variable_queue:len(VQ8), + VQ9 = variable_queue_publish(false, 1, VQ8), + VQ10 = variable_queue_set_ram_duration_target(0, VQ9), + {VQ11, _AckTags2} = variable_queue_fetch(Count1, true, true, Count, VQ10), + {VQ12, _AckTags3} = variable_queue_fetch(1, false, false, 1, VQ11), + VQ12. + +variable_queue_all_the_bits_not_covered_elsewhere_B(Config) -> + passed = rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, variable_queue_all_the_bits_not_covered_elsewhere_B1, [Config]). + +variable_queue_all_the_bits_not_covered_elsewhere_B1(Config) -> + with_fresh_variable_queue( + fun variable_queue_all_the_bits_not_covered_elsewhere_B2/2, + ?config(variable_queue_type, Config)). + +variable_queue_all_the_bits_not_covered_elsewhere_B2(VQ0, QName) -> + VQ1 = variable_queue_set_ram_duration_target(0, VQ0), + VQ2 = variable_queue_publish(false, 4, VQ1), + {VQ3, AckTags} = variable_queue_fetch(2, false, false, 4, VQ2), + {_Guids, VQ4} = + rabbit_variable_queue:requeue(AckTags, VQ3), + VQ5 = rabbit_variable_queue:timeout(VQ4), + _VQ6 = rabbit_variable_queue:terminate(shutdown, VQ5), + VQ7 = variable_queue_init(test_amqqueue(QName, true), true), + {empty, VQ8} = rabbit_variable_queue:fetch(false, VQ7), + VQ8. + +variable_queue_drop(Config) -> + passed = rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, variable_queue_drop1, [Config]). + +variable_queue_drop1(Config) -> + with_fresh_variable_queue( + fun variable_queue_drop2/2, + ?config(variable_queue_type, Config)). + +variable_queue_drop2(VQ0, _QName) -> + %% start by sending a messages + VQ1 = variable_queue_publish(false, 1, VQ0), + %% drop message with AckRequired = true + {{MsgId, AckTag}, VQ2} = rabbit_variable_queue:drop(true, VQ1), + true = rabbit_variable_queue:is_empty(VQ2), + true = AckTag =/= undefinded, + %% drop again -> empty + {empty, VQ3} = rabbit_variable_queue:drop(false, VQ2), + %% requeue + {[MsgId], VQ4} = rabbit_variable_queue:requeue([AckTag], VQ3), + %% drop message with AckRequired = false + {{MsgId, undefined}, VQ5} = rabbit_variable_queue:drop(false, VQ4), + true = rabbit_variable_queue:is_empty(VQ5), + VQ5. + +variable_queue_fold_msg_on_disk(Config) -> + passed = rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, variable_queue_fold_msg_on_disk1, [Config]). + +variable_queue_fold_msg_on_disk1(Config) -> + with_fresh_variable_queue( + fun variable_queue_fold_msg_on_disk2/2, + ?config(variable_queue_type, Config)). + +variable_queue_fold_msg_on_disk2(VQ0, _QName) -> + VQ1 = variable_queue_publish(true, 1, VQ0), + {VQ2, AckTags} = variable_queue_fetch(1, true, false, 1, VQ1), + {ok, VQ3} = rabbit_variable_queue:ackfold(fun (_M, _A, ok) -> ok end, + ok, VQ2, AckTags), + VQ3. + +variable_queue_dropfetchwhile(Config) -> + passed = rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, variable_queue_dropfetchwhile1, [Config]). + +variable_queue_dropfetchwhile1(Config) -> + with_fresh_variable_queue( + fun variable_queue_dropfetchwhile2/2, + ?config(variable_queue_type, Config)). + +variable_queue_dropfetchwhile2(VQ0, _QName) -> + Count = 10, + + %% add messages with sequential expiry + VQ1 = variable_queue_publish( + false, 1, Count, + fun (N, Props) -> Props#message_properties{expiry = N} end, + fun erlang:term_to_binary/1, VQ0), + + %% fetch the first 5 messages + {#message_properties{expiry = 6}, {Msgs, AckTags}, VQ2} = + rabbit_variable_queue:fetchwhile( + fun (#message_properties{expiry = Expiry}) -> Expiry =< 5 end, + fun (Msg, AckTag, {MsgAcc, AckAcc}) -> + {[Msg | MsgAcc], [AckTag | AckAcc]} + end, {[], []}, VQ1), + true = lists:seq(1, 5) == [msg2int(M) || M <- lists:reverse(Msgs)], + + %% requeue them + {_MsgIds, VQ3} = rabbit_variable_queue:requeue(AckTags, VQ2), + + %% drop the first 5 messages + {#message_properties{expiry = 6}, VQ4} = + rabbit_variable_queue:dropwhile( + fun (#message_properties {expiry = Expiry}) -> Expiry =< 5 end, VQ3), + + %% fetch 5 + VQ5 = lists:foldl(fun (N, VQN) -> + {{Msg, _, _}, VQM} = + rabbit_variable_queue:fetch(false, VQN), + true = msg2int(Msg) == N, + VQM + end, VQ4, lists:seq(6, Count)), + + %% should be empty now + true = rabbit_variable_queue:is_empty(VQ5), + + VQ5. + +variable_queue_dropwhile_varying_ram_duration(Config) -> + passed = rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, variable_queue_dropwhile_varying_ram_duration1, [Config]). + +variable_queue_dropwhile_varying_ram_duration1(Config) -> + with_fresh_variable_queue( + fun variable_queue_dropwhile_varying_ram_duration2/2, + ?config(variable_queue_type, Config)). + +variable_queue_dropwhile_varying_ram_duration2(VQ0, _QName) -> + test_dropfetchwhile_varying_ram_duration( + fun (VQ1) -> + {_, VQ2} = rabbit_variable_queue:dropwhile( + fun (_) -> false end, VQ1), + VQ2 + end, VQ0). + +variable_queue_fetchwhile_varying_ram_duration(Config) -> + passed = rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, variable_queue_fetchwhile_varying_ram_duration1, [Config]). + +variable_queue_fetchwhile_varying_ram_duration1(Config) -> + with_fresh_variable_queue( + fun variable_queue_fetchwhile_varying_ram_duration2/2, + ?config(variable_queue_type, Config)). + +variable_queue_fetchwhile_varying_ram_duration2(VQ0, _QName) -> + test_dropfetchwhile_varying_ram_duration( + fun (VQ1) -> + {_, ok, VQ2} = rabbit_variable_queue:fetchwhile( + fun (_) -> false end, + fun (_, _, A) -> A end, + ok, VQ1), + VQ2 + end, VQ0). + +test_dropfetchwhile_varying_ram_duration(Fun, VQ0) -> + VQ1 = variable_queue_publish(false, 1, VQ0), + VQ2 = variable_queue_set_ram_duration_target(0, VQ1), + VQ3 = Fun(VQ2), + VQ4 = variable_queue_set_ram_duration_target(infinity, VQ3), + VQ5 = variable_queue_publish(false, 1, VQ4), + VQ6 = Fun(VQ5), + VQ6. + +variable_queue_ack_limiting(Config) -> + passed = rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, variable_queue_ack_limiting1, [Config]). + +variable_queue_ack_limiting1(Config) -> + with_fresh_variable_queue( + fun variable_queue_ack_limiting2/2, + ?config(variable_queue_type, Config)). + +variable_queue_ack_limiting2(VQ0, _Config) -> + %% start by sending in a bunch of messages + Len = 1024, + VQ1 = variable_queue_publish(false, Len, VQ0), + + %% squeeze and relax queue + Churn = Len div 32, + VQ2 = publish_fetch_and_ack(Churn, Len, VQ1), + + %% update stats for duration + {_Duration, VQ3} = rabbit_variable_queue:ram_duration(VQ2), + + %% fetch half the messages + {VQ4, _AckTags} = variable_queue_fetch(Len div 2, false, false, Len, VQ3), + + VQ5 = check_variable_queue_status( + VQ4, [{len, Len div 2}, + {messages_unacknowledged_ram, Len div 2}, + {messages_ready_ram, Len div 2}, + {messages_ram, Len}]), + + %% ensure all acks go to disk on 0 duration target + VQ6 = check_variable_queue_status( + variable_queue_set_ram_duration_target(0, VQ5), + [{len, Len div 2}, + {target_ram_count, 0}, + {messages_unacknowledged_ram, 0}, + {messages_ready_ram, 0}, + {messages_ram, 0}]), + + VQ6. + +variable_queue_purge(Config) -> + passed = rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, variable_queue_purge1, [Config]). + +variable_queue_purge1(Config) -> + with_fresh_variable_queue( + fun variable_queue_purge2/2, + ?config(variable_queue_type, Config)). + +variable_queue_purge2(VQ0, _Config) -> + LenDepth = fun (VQ) -> + {rabbit_variable_queue:len(VQ), + rabbit_variable_queue:depth(VQ)} + end, + VQ1 = variable_queue_publish(false, 10, VQ0), + {VQ2, Acks} = variable_queue_fetch(6, false, false, 10, VQ1), + {4, VQ3} = rabbit_variable_queue:purge(VQ2), + {0, 6} = LenDepth(VQ3), + {_, VQ4} = rabbit_variable_queue:requeue(lists:sublist(Acks, 2), VQ3), + {2, 6} = LenDepth(VQ4), + VQ5 = rabbit_variable_queue:purge_acks(VQ4), + {2, 2} = LenDepth(VQ5), + VQ5. + +variable_queue_requeue(Config) -> + passed = rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, variable_queue_requeue1, [Config]). + +variable_queue_requeue1(Config) -> + with_fresh_variable_queue( + fun variable_queue_requeue2/2, + ?config(variable_queue_type, Config)). + +variable_queue_requeue2(VQ0, _Config) -> + {_PendingMsgs, RequeuedMsgs, FreshMsgs, VQ1} = + variable_queue_with_holes(VQ0), + Msgs = + lists:zip(RequeuedMsgs, + lists:duplicate(length(RequeuedMsgs), true)) ++ + lists:zip(FreshMsgs, + lists:duplicate(length(FreshMsgs), false)), + VQ2 = lists:foldl(fun ({I, Requeued}, VQa) -> + {{M, MRequeued, _}, VQb} = + rabbit_variable_queue:fetch(true, VQa), + Requeued = MRequeued, %% assertion + I = msg2int(M), %% assertion + VQb + end, VQ1, Msgs), + {empty, VQ3} = rabbit_variable_queue:fetch(true, VQ2), + VQ3. + +%% requeue from ram_pending_ack into q3, move to delta and then empty queue +variable_queue_requeue_ram_beta(Config) -> + passed = rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, variable_queue_requeue_ram_beta1, [Config]). + +variable_queue_requeue_ram_beta1(Config) -> + with_fresh_variable_queue( + fun variable_queue_requeue_ram_beta2/2, + ?config(variable_queue_type, Config)). + +variable_queue_requeue_ram_beta2(VQ0, _Config) -> + Count = rabbit_queue_index:next_segment_boundary(0)*2 + 2, + VQ1 = variable_queue_publish(false, Count, VQ0), + {VQ2, AcksR} = variable_queue_fetch(Count, false, false, Count, VQ1), + {Back, Front} = lists:split(Count div 2, AcksR), + {_, VQ3} = rabbit_variable_queue:requeue(erlang:tl(Back), VQ2), + VQ4 = variable_queue_set_ram_duration_target(0, VQ3), + {_, VQ5} = rabbit_variable_queue:requeue([erlang:hd(Back)], VQ4), + VQ6 = requeue_one_by_one(Front, VQ5), + {VQ7, AcksAll} = variable_queue_fetch(Count, false, true, Count, VQ6), + {_, VQ8} = rabbit_variable_queue:ack(AcksAll, VQ7), + VQ8. + +variable_queue_fold(Config) -> + passed = rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, variable_queue_fold1, [Config]). + +variable_queue_fold1(Config) -> + with_fresh_variable_queue( + fun variable_queue_fold2/2, + ?config(variable_queue_type, Config)). + +variable_queue_fold2(VQ0, _Config) -> + {PendingMsgs, RequeuedMsgs, FreshMsgs, VQ1} = + variable_queue_with_holes(VQ0), + Count = rabbit_variable_queue:depth(VQ1), + Msgs = lists:sort(PendingMsgs ++ RequeuedMsgs ++ FreshMsgs), + lists:foldl(fun (Cut, VQ2) -> + test_variable_queue_fold(Cut, Msgs, PendingMsgs, VQ2) + end, VQ1, [0, 1, 2, Count div 2, + Count - 1, Count, Count + 1, Count * 2]). + +test_variable_queue_fold(Cut, Msgs, PendingMsgs, VQ0) -> + {Acc, VQ1} = rabbit_variable_queue:fold( + fun (M, _, Pending, A) -> + MInt = msg2int(M), + Pending = lists:member(MInt, PendingMsgs), %% assert + case MInt =< Cut of + true -> {cont, [MInt | A]}; + false -> {stop, A} + end + end, [], VQ0), + Expected = lists:takewhile(fun (I) -> I =< Cut end, Msgs), + Expected = lists:reverse(Acc), %% assertion + VQ1. + +variable_queue_batch_publish(Config) -> + passed = rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, variable_queue_batch_publish1, [Config]). + +variable_queue_batch_publish1(Config) -> + with_fresh_variable_queue( + fun variable_queue_batch_publish2/2, + ?config(variable_queue_type, Config)). + +variable_queue_batch_publish2(VQ, _Config) -> + Count = 10, + VQ1 = variable_queue_batch_publish(true, Count, VQ), + Count = rabbit_variable_queue:len(VQ1), + VQ1. + +variable_queue_batch_publish_delivered(Config) -> + passed = rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, variable_queue_batch_publish_delivered1, [Config]). + +variable_queue_batch_publish_delivered1(Config) -> + with_fresh_variable_queue( + fun variable_queue_batch_publish_delivered2/2, + ?config(variable_queue_type, Config)). + +variable_queue_batch_publish_delivered2(VQ, _Config) -> + Count = 10, + VQ1 = variable_queue_batch_publish_delivered(true, Count, VQ), + Count = rabbit_variable_queue:depth(VQ1), + VQ1. + +%% same as test_variable_queue_requeue_ram_beta but randomly changing +%% the queue mode after every step. +variable_queue_mode_change(Config) -> + passed = rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, variable_queue_mode_change1, [Config]). + +variable_queue_mode_change1(Config) -> + with_fresh_variable_queue( + fun variable_queue_mode_change2/2, + ?config(variable_queue_type, Config)). + +variable_queue_mode_change2(VQ0, _Config) -> + Count = rabbit_queue_index:next_segment_boundary(0)*2 + 2, + VQ1 = variable_queue_publish(false, Count, VQ0), + VQ2 = maybe_switch_queue_mode(VQ1), + {VQ3, AcksR} = variable_queue_fetch(Count, false, false, Count, VQ2), + VQ4 = maybe_switch_queue_mode(VQ3), + {Back, Front} = lists:split(Count div 2, AcksR), + {_, VQ5} = rabbit_variable_queue:requeue(erlang:tl(Back), VQ4), + VQ6 = maybe_switch_queue_mode(VQ5), + VQ7 = variable_queue_set_ram_duration_target(0, VQ6), + VQ8 = maybe_switch_queue_mode(VQ7), + {_, VQ9} = rabbit_variable_queue:requeue([erlang:hd(Back)], VQ8), + VQ10 = maybe_switch_queue_mode(VQ9), + VQ11 = requeue_one_by_one(Front, VQ10), + VQ12 = maybe_switch_queue_mode(VQ11), + {VQ13, AcksAll} = variable_queue_fetch(Count, false, true, Count, VQ12), + VQ14 = maybe_switch_queue_mode(VQ13), + {_, VQ15} = rabbit_variable_queue:ack(AcksAll, VQ14), + VQ16 = maybe_switch_queue_mode(VQ15), + VQ16. + +maybe_switch_queue_mode(VQ) -> + Mode = random_queue_mode(), + set_queue_mode(Mode, VQ). + +random_queue_mode() -> + Modes = [lazy, default], + lists:nth(rand:uniform(length(Modes)), Modes). + +pub_res({_, VQS}) -> + VQS; +pub_res(VQS) -> + VQS. + +make_publish(IsPersistent, PayloadFun, PropFun, N) -> + {rabbit_basic:message( + rabbit_misc:r(<<>>, exchange, <<>>), + <<>>, #'P_basic'{delivery_mode = case IsPersistent of + true -> 2; + false -> 1 + end}, + PayloadFun(N)), + PropFun(N, #message_properties{size = 10}), + false}. + +make_publish_delivered(IsPersistent, PayloadFun, PropFun, N) -> + {rabbit_basic:message( + rabbit_misc:r(<<>>, exchange, <<>>), + <<>>, #'P_basic'{delivery_mode = case IsPersistent of + true -> 2; + false -> 1 + end}, + PayloadFun(N)), + PropFun(N, #message_properties{size = 10})}. + +queue_name(Config, Name) -> + Name1 = iolist_to_binary(rabbit_ct_helpers:config_to_testcase_name(Config, Name)), + queue_name(Name1). + +queue_name(Name) -> + rabbit_misc:r(<<"/">>, queue, Name). + +test_queue() -> + queue_name(rabbit_guid:gen()). + +init_test_queue(QName) -> + PRef = rabbit_guid:gen(), + PersistentClient = msg_store_client_init(?PERSISTENT_MSG_STORE, PRef), + Res = rabbit_queue_index:recover( + QName, [], false, + fun (MsgId) -> + rabbit_msg_store:contains(MsgId, PersistentClient) + end, + fun nop/1, fun nop/1), + ok = rabbit_msg_store:client_delete_and_terminate(PersistentClient), + Res. + +restart_test_queue(Qi, QName) -> + _ = rabbit_queue_index:terminate(?VHOST, [], Qi), + ok = rabbit_variable_queue:stop(?VHOST), + {ok, _} = rabbit_variable_queue:start(?VHOST, [QName]), + init_test_queue(QName). + +empty_test_queue(QName) -> + ok = rabbit_variable_queue:stop(?VHOST), + {ok, _} = rabbit_variable_queue:start(?VHOST, []), + {0, 0, Qi} = init_test_queue(QName), + _ = rabbit_queue_index:delete_and_terminate(Qi), + ok. + +unin_empty_test_queue(QName) -> + {0, 0, Qi} = init_test_queue(QName), + _ = rabbit_queue_index:delete_and_terminate(Qi), + ok. + +with_empty_test_queue(Fun) -> + QName = test_queue(), + ok = empty_test_queue(QName), + {0, 0, Qi} = init_test_queue(QName), + rabbit_queue_index:delete_and_terminate(Fun(Qi, QName)). + +restart_app() -> + rabbit:stop(), + rabbit:start(). + +queue_index_publish(SeqIds, Persistent, Qi) -> + Ref = rabbit_guid:gen(), + MsgStore = case Persistent of + true -> ?PERSISTENT_MSG_STORE; + false -> ?TRANSIENT_MSG_STORE + end, + MSCState = msg_store_client_init(MsgStore, Ref), + {A, B = [{_SeqId, LastMsgIdWritten} | _]} = + lists:foldl( + fun (SeqId, {QiN, SeqIdsMsgIdsAcc}) -> + MsgId = rabbit_guid:gen(), + QiM = rabbit_queue_index:publish( + MsgId, SeqId, #message_properties{size = 10}, + Persistent, infinity, QiN), + ok = rabbit_msg_store:write(MsgId, MsgId, MSCState), + {QiM, [{SeqId, MsgId} | SeqIdsMsgIdsAcc]} + end, {Qi, []}, SeqIds), + %% do this just to force all of the publishes through to the msg_store: + true = rabbit_msg_store:contains(LastMsgIdWritten, MSCState), + ok = rabbit_msg_store:client_delete_and_terminate(MSCState), + {A, B}. + +verify_read_with_published(_Delivered, _Persistent, [], _) -> + ok; +verify_read_with_published(Delivered, Persistent, + [{MsgId, SeqId, _Props, Persistent, Delivered}|Read], + [{SeqId, MsgId}|Published]) -> + verify_read_with_published(Delivered, Persistent, Read, Published); +verify_read_with_published(_Delivered, _Persistent, _Read, _Published) -> + ko. + +nop(_) -> ok. +nop(_, _) -> ok. + +msg_store_client_init(MsgStore, Ref) -> + rabbit_vhost_msg_store:client_init(?VHOST, MsgStore, Ref, undefined, undefined). + +variable_queue_init(Q, Recover) -> + rabbit_variable_queue:init( + Q, case Recover of + true -> non_clean_shutdown; + false -> new + end, fun nop/2, fun nop/2, fun nop/1, fun nop/1). + +publish_and_confirm(Q, Payload, Count) -> + Seqs = lists:seq(1, Count), + QTState0 = rabbit_queue_type:new(Q, rabbit_queue_type:init()), + QTState = + lists:foldl( + fun (Seq, Acc0) -> + Msg = rabbit_basic:message(rabbit_misc:r(<<>>, exchange, <<>>), + <<>>, #'P_basic'{delivery_mode = 2}, + Payload), + Delivery = #delivery{mandatory = false, sender = self(), + confirm = true, message = Msg, msg_seq_no = Seq, + flow = noflow}, + {ok, Acc, _Actions} = rabbit_queue_type:deliver([Q], Delivery, Acc0), + Acc + end, QTState0, Seqs), + wait_for_confirms(gb_sets:from_list(Seqs)), + QTState. + +wait_for_confirms(Unconfirmed) -> + case gb_sets:is_empty(Unconfirmed) of + true -> ok; + false -> + receive + {'$gen_cast', + {queue_event, _QName, {confirm, Confirmed, _}}} -> + wait_for_confirms( + rabbit_misc:gb_sets_difference( + Unconfirmed, gb_sets:from_list(Confirmed))); + {'$gen_cast', {confirm, Confirmed, _}} -> + wait_for_confirms( + rabbit_misc:gb_sets_difference( + Unconfirmed, gb_sets:from_list(Confirmed))) + after ?TIMEOUT -> + flush(), + exit(timeout_waiting_for_confirm) + end + end. + +with_fresh_variable_queue(Fun, Mode) -> + Ref = make_ref(), + Me = self(), + %% Run in a separate process since rabbit_msg_store will send + %% bump_credit messages and we want to ignore them + spawn_link(fun() -> + QName = test_queue(), + ok = unin_empty_test_queue(QName), + VQ = variable_queue_init(test_amqqueue(QName, true), false), + S0 = variable_queue_status(VQ), + assert_props(S0, [{q1, 0}, {q2, 0}, + {delta, + {delta, undefined, 0, undefined}}, + {q3, 0}, {q4, 0}, + {len, 0}]), + VQ1 = set_queue_mode(Mode, VQ), + try + _ = rabbit_variable_queue:delete_and_terminate( + shutdown, Fun(VQ1, QName)), + Me ! Ref + catch + Type:Error:Stacktrace -> + Me ! {Ref, Type, Error, Stacktrace} + end + end), + receive + Ref -> ok; + {Ref, Type, Error, ST} -> exit({Type, Error, ST}) + end, + passed. + +set_queue_mode(Mode, VQ) -> + VQ1 = rabbit_variable_queue:set_queue_mode(Mode, VQ), + S1 = variable_queue_status(VQ1), + assert_props(S1, [{mode, Mode}]), + VQ1. + +variable_queue_publish(IsPersistent, Count, VQ) -> + variable_queue_publish(IsPersistent, Count, fun (_N, P) -> P end, VQ). + +variable_queue_publish(IsPersistent, Count, PropFun, VQ) -> + variable_queue_publish(IsPersistent, 1, Count, PropFun, + fun (_N) -> <<>> end, VQ). + +variable_queue_publish(IsPersistent, Start, Count, PropFun, PayloadFun, VQ) -> + variable_queue_wait_for_shuffling_end( + lists:foldl( + fun (N, VQN) -> + + rabbit_variable_queue:publish( + rabbit_basic:message( + rabbit_misc:r(<<>>, exchange, <<>>), + <<>>, #'P_basic'{delivery_mode = case IsPersistent of + true -> 2; + false -> 1 + end}, + PayloadFun(N)), + PropFun(N, #message_properties{size = 10}), + false, self(), noflow, VQN) + end, VQ, lists:seq(Start, Start + Count - 1))). + +variable_queue_batch_publish(IsPersistent, Count, VQ) -> + variable_queue_batch_publish(IsPersistent, Count, fun (_N, P) -> P end, VQ). + +variable_queue_batch_publish(IsPersistent, Count, PropFun, VQ) -> + variable_queue_batch_publish(IsPersistent, 1, Count, PropFun, + fun (_N) -> <<>> end, VQ). + +variable_queue_batch_publish(IsPersistent, Start, Count, PropFun, PayloadFun, VQ) -> + variable_queue_batch_publish0(IsPersistent, Start, Count, PropFun, + PayloadFun, fun make_publish/4, + fun rabbit_variable_queue:batch_publish/4, + VQ). + +variable_queue_batch_publish_delivered(IsPersistent, Count, VQ) -> + variable_queue_batch_publish_delivered(IsPersistent, Count, fun (_N, P) -> P end, VQ). + +variable_queue_batch_publish_delivered(IsPersistent, Count, PropFun, VQ) -> + variable_queue_batch_publish_delivered(IsPersistent, 1, Count, PropFun, + fun (_N) -> <<>> end, VQ). + +variable_queue_batch_publish_delivered(IsPersistent, Start, Count, PropFun, PayloadFun, VQ) -> + variable_queue_batch_publish0(IsPersistent, Start, Count, PropFun, + PayloadFun, fun make_publish_delivered/4, + fun rabbit_variable_queue:batch_publish_delivered/4, + VQ). + +variable_queue_batch_publish0(IsPersistent, Start, Count, PropFun, PayloadFun, + MakePubFun, PubFun, VQ) -> + Publishes = + [MakePubFun(IsPersistent, PayloadFun, PropFun, N) + || N <- lists:seq(Start, Start + Count - 1)], + Res = PubFun(Publishes, self(), noflow, VQ), + VQ1 = pub_res(Res), + variable_queue_wait_for_shuffling_end(VQ1). + +variable_queue_fetch(Count, IsPersistent, IsDelivered, Len, VQ) -> + lists:foldl(fun (N, {VQN, AckTagsAcc}) -> + Rem = Len - N, + {{#basic_message { is_persistent = IsPersistent }, + IsDelivered, AckTagN}, VQM} = + rabbit_variable_queue:fetch(true, VQN), + Rem = rabbit_variable_queue:len(VQM), + {VQM, [AckTagN | AckTagsAcc]} + end, {VQ, []}, lists:seq(1, Count)). + +test_amqqueue(QName, Durable) -> + rabbit_amqqueue:pseudo_queue(QName, self(), Durable). + +assert_prop(List, Prop, Value) -> + case proplists:get_value(Prop, List)of + Value -> ok; + _ -> {exit, Prop, exp, Value, List} + end. + +assert_props(List, PropVals) -> + [assert_prop(List, Prop, Value) || {Prop, Value} <- PropVals]. + +variable_queue_set_ram_duration_target(Duration, VQ) -> + variable_queue_wait_for_shuffling_end( + rabbit_variable_queue:set_ram_duration_target(Duration, VQ)). + +publish_fetch_and_ack(0, _Len, VQ0) -> + VQ0; +publish_fetch_and_ack(N, Len, VQ0) -> + VQ1 = variable_queue_publish(false, 1, VQ0), + {{_Msg, false, AckTag}, VQ2} = rabbit_variable_queue:fetch(true, VQ1), + Len = rabbit_variable_queue:len(VQ2), + {_Guids, VQ3} = rabbit_variable_queue:ack([AckTag], VQ2), + publish_fetch_and_ack(N-1, Len, VQ3). + +variable_queue_status(VQ) -> + Keys = rabbit_backing_queue:info_keys() -- [backing_queue_status], + [{K, rabbit_variable_queue:info(K, VQ)} || K <- Keys] ++ + rabbit_variable_queue:info(backing_queue_status, VQ). + +variable_queue_wait_for_shuffling_end(VQ) -> + case credit_flow:blocked() of + false -> VQ; + true -> + receive + {bump_credit, Msg} -> + credit_flow:handle_bump_msg(Msg), + variable_queue_wait_for_shuffling_end( + rabbit_variable_queue:resume(VQ)) + end + end. + +msg2int(#basic_message{content = #content{ payload_fragments_rev = P}}) -> + binary_to_term(list_to_binary(lists:reverse(P))). + +ack_subset(AckSeqs, Interval, Rem) -> + lists:filter(fun ({_Ack, N}) -> (N + Rem) rem Interval == 0 end, AckSeqs). + +requeue_one_by_one(Acks, VQ) -> + lists:foldl(fun (AckTag, VQN) -> + {_MsgId, VQM} = rabbit_variable_queue:requeue( + [AckTag], VQN), + VQM + end, VQ, Acks). + +%% Create a vq with messages in q1, delta, and q3, and holes (in the +%% form of pending acks) in the latter two. +variable_queue_with_holes(VQ0) -> + Interval = 2048, %% should match vq:IO_BATCH_SIZE + Count = rabbit_queue_index:next_segment_boundary(0)*2 + 2 * Interval, + Seq = lists:seq(1, Count), + VQ1 = variable_queue_set_ram_duration_target(0, VQ0), + VQ2 = variable_queue_publish( + false, 1, Count, + fun (_, P) -> P end, fun erlang:term_to_binary/1, VQ1), + {VQ3, AcksR} = variable_queue_fetch(Count, false, false, Count, VQ2), + Acks = lists:reverse(AcksR), + AckSeqs = lists:zip(Acks, Seq), + [{Subset1, _Seq1}, {Subset2, _Seq2}, {Subset3, Seq3}] = + [lists:unzip(ack_subset(AckSeqs, Interval, I)) || I <- [0, 1, 2]], + %% we requeue in three phases in order to exercise requeuing logic + %% in various vq states + {_MsgIds, VQ4} = rabbit_variable_queue:requeue( + Acks -- (Subset1 ++ Subset2 ++ Subset3), VQ3), + VQ5 = requeue_one_by_one(Subset1, VQ4), + %% by now we have some messages (and holes) in delta + VQ6 = requeue_one_by_one(Subset2, VQ5), + VQ7 = variable_queue_set_ram_duration_target(infinity, VQ6), + %% add the q1 tail + VQ8 = variable_queue_publish( + true, Count + 1, Interval, + fun (_, P) -> P end, fun erlang:term_to_binary/1, VQ7), + %% assertions + Status = variable_queue_status(VQ8), + + vq_with_holes_assertions(VQ8, proplists:get_value(mode, Status)), + Depth = Count + Interval, + Depth = rabbit_variable_queue:depth(VQ8), + Len = Depth - length(Subset3), + Len = rabbit_variable_queue:len(VQ8), + + {Seq3, Seq -- Seq3, lists:seq(Count + 1, Count + Interval), VQ8}. + +vq_with_holes_assertions(VQ, default) -> + [false = + case V of + {delta, _, 0, _} -> true; + 0 -> true; + _ -> false + end || {K, V} <- variable_queue_status(VQ), + lists:member(K, [q1, delta, q3])]; +vq_with_holes_assertions(VQ, lazy) -> + [false = + case V of + {delta, _, 0, _} -> true; + _ -> false + end || {K, V} <- variable_queue_status(VQ), + lists:member(K, [delta])]. + +check_variable_queue_status(VQ0, Props) -> + VQ1 = variable_queue_wait_for_shuffling_end(VQ0), + S = variable_queue_status(VQ1), + assert_props(S, Props), + VQ1. + +flush() -> + receive + Any -> + ct:pal("flush ~p", [Any]), + flush() + after 0 -> + ok + end. diff --git a/deps/rabbit/test/channel_interceptor_SUITE.erl b/deps/rabbit/test/channel_interceptor_SUITE.erl new file mode 100644 index 0000000000..e0a8050598 --- /dev/null +++ b/deps/rabbit/test/channel_interceptor_SUITE.erl @@ -0,0 +1,108 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2016-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(channel_interceptor_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). + +-compile(export_all). + +all() -> + [ + {group, non_parallel_tests} + ]. + +groups() -> + [ + {non_parallel_tests, [], [ + register_interceptor, + register_failing_interceptors + ]} + ]. + +%% ------------------------------------------------------------------- +%% Testsuite setup/teardown. +%% ------------------------------------------------------------------- + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(_, Config) -> + Config. + +end_per_group(_, Config) -> + Config. + +init_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_started(Config, Testcase), + Config1 = rabbit_ct_helpers:set_config(Config, [ + {rmq_nodename_suffix, Testcase} + ]), + rabbit_ct_helpers:run_steps(Config1, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()). + +end_per_testcase(Testcase, Config) -> + Config1 = rabbit_ct_helpers:run_steps(Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()), + rabbit_ct_helpers:testcase_finished(Config1, Testcase). + +%% ------------------------------------------------------------------- +%% Testcases. +%% ------------------------------------------------------------------- + +register_interceptor(Config) -> + passed = rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, register_interceptor1, [Config, dummy_interceptor]). + +register_interceptor1(Config, Interceptor) -> + PredefinedChannels = rabbit_channel:list(), + + Ch1 = rabbit_ct_client_helpers:open_channel(Config, 0), + + QName = <<"register_interceptor-q">>, + amqp_channel:call(Ch1, #'queue.declare'{queue = QName}), + + [ChannelProc] = rabbit_channel:list() -- PredefinedChannels, + + [{interceptors, []}] = rabbit_channel:info(ChannelProc, [interceptors]), + + check_send_receive(Ch1, QName, <<"bar">>, <<"bar">>), + + ok = rabbit_registry:register(channel_interceptor, + <<"dummy interceptor">>, + Interceptor), + [{interceptors, [{Interceptor, undefined}]}] = + rabbit_channel:info(ChannelProc, [interceptors]), + + check_send_receive(Ch1, QName, <<"bar">>, <<"">>), + + ok = rabbit_registry:unregister(channel_interceptor, + <<"dummy interceptor">>), + [{interceptors, []}] = rabbit_channel:info(ChannelProc, [interceptors]), + + check_send_receive(Ch1, QName, <<"bar">>, <<"bar">>), + passed. + +register_failing_interceptors(Config) -> + passed = rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, register_interceptor1, [Config, failing_dummy_interceptor]). + +check_send_receive(Ch1, QName, Send, Receive) -> + amqp_channel:call(Ch1, + #'basic.publish'{routing_key = QName}, + #amqp_msg{payload = Send}), + + {#'basic.get_ok'{}, #amqp_msg{payload = Receive}} = + amqp_channel:call(Ch1, #'basic.get'{queue = QName, + no_ack = true}). diff --git a/deps/rabbit/test/channel_operation_timeout_SUITE.erl b/deps/rabbit/test/channel_operation_timeout_SUITE.erl new file mode 100644 index 0000000000..15e0188604 --- /dev/null +++ b/deps/rabbit/test/channel_operation_timeout_SUITE.erl @@ -0,0 +1,198 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(channel_operation_timeout_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). +-include("amqqueue.hrl"). + +-compile([export_all]). + +-import(rabbit_misc, [pget/2]). + +-define(CONFIG, [cluster_ab]). +-define(DEFAULT_VHOST, <<"/">>). +-define(QRESOURCE(Q), rabbit_misc:r(?DEFAULT_VHOST, queue, Q)). +-define(TIMEOUT_TEST_MSG, <<"timeout_test_msg!">>). +-define(DELAY, 25). + +all() -> + [ + notify_down_all + ]. + +%% ------------------------------------------------------------------- +%% Testsuite setup/teardown. +%% ------------------------------------------------------------------- + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(_, Config) -> + Config. + +end_per_group(_, Config) -> + Config. + +init_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_started(Config, Testcase), + ClusterSize = 2, + TestNumber = rabbit_ct_helpers:testcase_number(Config, ?MODULE, Testcase), + Config1 = rabbit_ct_helpers:set_config(Config, [ + {rmq_nodes_count, ClusterSize}, + {rmq_nodename_suffix, Testcase}, + {tcp_ports_base, {skip_n_nodes, TestNumber * ClusterSize}} + ]), + rabbit_ct_helpers:run_steps(Config1, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()). + +end_per_testcase(Testcase, Config) -> + Config1 = rabbit_ct_helpers:run_steps(Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()), + rabbit_ct_helpers:testcase_finished(Config1, Testcase). + +%% ------------------------------------------------------------------- +%% Testcases. +%% ------------------------------------------------------------------- + +notify_down_all(Config) -> + Rabbit = rabbit_ct_broker_helpers:get_node_config(Config, 0, nodename), + RabbitCh = rabbit_ct_client_helpers:open_channel(Config, 0), + HareCh = rabbit_ct_client_helpers:open_channel(Config, 1), + + ct:pal("one"), + %% success + set_channel_operation_timeout_config(Config, 1000), + configure_bq(Config), + QCfg0 = qconfig(RabbitCh, <<"q0">>, <<"ex0">>, true, false), + declare(QCfg0), + ct:pal("two"), + %% Testing rabbit_amqqueue:notify_down_all via rabbit_channel. + %% Consumer count = 0 after correct channel termination and + %% notification of queues via delegate:call/3 + true = (0 =/= length(get_consumers(Config, Rabbit, ?DEFAULT_VHOST))), + rabbit_ct_client_helpers:close_channel(RabbitCh), + 0 = length(get_consumers(Config, Rabbit, ?DEFAULT_VHOST)), + false = is_process_alive(RabbitCh), + ct:pal("three"), + + %% fail + set_channel_operation_timeout_config(Config, 10), + QCfg2 = qconfig(HareCh, <<"q1">>, <<"ex1">>, true, false), + declare(QCfg2), + publish(QCfg2, ?TIMEOUT_TEST_MSG), + timer:sleep(?DELAY), + rabbit_ct_client_helpers:close_channel(HareCh), + timer:sleep(?DELAY), + false = is_process_alive(HareCh), + + pass. + +%% ------------------------- +%% Internal helper functions +%% ------------------------- + +set_channel_operation_timeout_config(Config, Timeout) -> + [ok = Ret + || Ret <- rabbit_ct_broker_helpers:rpc_all(Config, + application, set_env, [rabbit, channel_operation_timeout, Timeout])], + ok. + +set_channel_operation_backing_queue(Config) -> + [ok = Ret + || Ret <- rabbit_ct_broker_helpers:rpc_all(Config, + application, set_env, + [rabbit, backing_queue_module, channel_operation_timeout_test_queue])], + ok. + +re_enable_priority_queue(Config) -> + [ok = Ret + || Ret <- rabbit_ct_broker_helpers:rpc_all(Config, + rabbit_priority_queue, enable, [])], + ok. + +declare(QCfg) -> + QDeclare = #'queue.declare'{queue = Q = pget(name, QCfg), durable = true}, + #'queue.declare_ok'{} = amqp_channel:call(Ch = pget(ch, QCfg), QDeclare), + + ExDeclare = #'exchange.declare'{exchange = Ex = pget(ex, QCfg)}, + #'exchange.declare_ok'{} = amqp_channel:call(Ch, ExDeclare), + + #'queue.bind_ok'{} = + amqp_channel:call(Ch, #'queue.bind'{queue = Q, + exchange = Ex, + routing_key = Q}), + maybe_subscribe(QCfg). + +maybe_subscribe(QCfg) -> + case pget(consume, QCfg) of + true -> + Sub = #'basic.consume'{queue = pget(name, QCfg)}, + Ch = pget(ch, QCfg), + Del = pget(deliver, QCfg), + amqp_channel:subscribe(Ch, Sub, + spawn(fun() -> consume(Ch, Del) end)); + _ -> ok + end. + +consume(_Ch, false) -> receive_nothing(); +consume(Ch, Deliver = true) -> + receive + {#'basic.deliver'{}, _Msg} -> + consume(Ch, Deliver) + end. + +publish(QCfg, Msg) -> + Publish = #'basic.publish'{exchange = pget(ex, QCfg), + routing_key = pget(name, QCfg)}, + amqp_channel:call(pget(ch, QCfg), Publish, + #amqp_msg{payload = Msg}). + +get_consumers(Config, Node, VHost) when is_atom(Node), + is_binary(VHost) -> + rabbit_ct_broker_helpers:rpc(Config, Node, + rabbit_amqqueue, consumers_all, [VHost]). + +get_amqqueue(QName0, []) -> + throw({not_found, QName0}); +get_amqqueue(QName0, [Q | Rem]) when ?is_amqqueue(Q) -> + QName1 = amqqueue:get_name(Q), + compare_amqqueue(QName0, QName1, Q, Rem). + +compare_amqqueue(QName, QName, Q, _Rem) -> + Q; +compare_amqqueue(QName, _, _, Rem) -> + get_amqqueue(QName, Rem). + +qconfig(Ch, Name, Ex, Consume, Deliver) -> + [{ch, Ch}, {name, Name}, {ex,Ex}, {consume, Consume}, {deliver, Deliver}]. + +receive_nothing() -> + receive + after infinity -> void + end. + +unhandled_req(Fun) -> + try + Fun() + catch + exit:{{shutdown,{_, ?NOT_FOUND, _}}, _} -> ok; + _:Reason -> {error, Reason} + end. + +configure_bq(Config) -> + ok = set_channel_operation_backing_queue(Config), + ok = re_enable_priority_queue(Config), + ok = rabbit_ct_broker_helpers:add_code_path_to_all_nodes(Config, + ?MODULE). diff --git a/deps/rabbit/test/channel_operation_timeout_test_queue.erl b/deps/rabbit/test/channel_operation_timeout_test_queue.erl new file mode 100644 index 0000000000..3190dad7a8 --- /dev/null +++ b/deps/rabbit/test/channel_operation_timeout_test_queue.erl @@ -0,0 +1,323 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(channel_operation_timeout_test_queue). + +-export([init/3, terminate/2, delete_and_terminate/2, delete_crashed/1, + purge/1, purge_acks/1, + publish/6, publish_delivered/5, + batch_publish/4, batch_publish_delivered/4, + discard/4, drain_confirmed/1, + dropwhile/2, fetchwhile/4, fetch/2, drop/2, ack/2, requeue/2, + ackfold/4, fold/3, len/1, is_empty/1, depth/1, + set_ram_duration_target/2, ram_duration/1, needs_timeout/1, timeout/1, + handle_pre_hibernate/1, resume/1, msg_rates/1, + info/2, invoke/3, is_duplicate/2, set_queue_mode/2, + start/2, stop/1, zip_msgs_and_acks/4, handle_info/2]). + +%%---------------------------------------------------------------------------- +%% This test backing queue follows the variable queue implementation, with +%% the exception that it will introduce infinite delays on some operations if +%% the test message has been published, and is awaiting acknowledgement in the +%% queue index. Test message is "timeout_test_msg!". +%% +%%---------------------------------------------------------------------------- + +-behaviour(rabbit_backing_queue). + +-record(vqstate, + { q1, + q2, + delta, + q3, + q4, + next_seq_id, + ram_pending_ack, %% msgs using store, still in RAM + disk_pending_ack, %% msgs in store, paged out + qi_pending_ack, %% msgs using qi, *can't* be paged out + index_state, + msg_store_clients, + durable, + transient_threshold, + qi_embed_msgs_below, + + len, %% w/o unacked + bytes, %% w/o unacked + unacked_bytes, + persistent_count, %% w unacked + persistent_bytes, %% w unacked + delta_transient_bytes, %% + + target_ram_count, + ram_msg_count, %% w/o unacked + ram_msg_count_prev, + ram_ack_count_prev, + ram_bytes, %% w unacked + out_counter, + in_counter, + rates, + msgs_on_disk, + msg_indices_on_disk, + unconfirmed, + confirmed, + ack_out_counter, + ack_in_counter, + %% Unlike the other counters these two do not feed into + %% #rates{} and get reset + disk_read_count, + disk_write_count, + + io_batch_size, + + %% default queue or lazy queue + mode, + %% number of reduce_memory_usage executions, once it + %% reaches a threshold the queue will manually trigger a runtime GC + %% see: maybe_execute_gc/1 + memory_reduction_run_count, + %% Queue data is grouped by VHost. We need to store it + %% to work with queue index. + virtual_host, + waiting_bump = false + }). + +-record(rates, { in, out, ack_in, ack_out, timestamp }). + +-record(msg_status, + { seq_id, + msg_id, + msg, + is_persistent, + is_delivered, + msg_in_store, + index_on_disk, + persist_to, + msg_props + }). + +-record(delta, + { start_seq_id, %% start_seq_id is inclusive + count, + transient, + end_seq_id %% end_seq_id is exclusive + }). + + +-include("rabbit.hrl"). +-include("rabbit_framing.hrl"). + +-define(QUEUE, lqueue). +-define(TIMEOUT_TEST_MSG, <<"timeout_test_msg!">>). + +%%---------------------------------------------------------------------------- + +-type seq_id() :: non_neg_integer(). + +-type rates() :: #rates { in :: float(), + out :: float(), + ack_in :: float(), + ack_out :: float(), + timestamp :: rabbit_types:timestamp()}. + +-type delta() :: #delta { start_seq_id :: non_neg_integer(), + count :: non_neg_integer(), + end_seq_id :: non_neg_integer() }. + +%% The compiler (rightfully) complains that ack() and state() are +%% unused. For this reason we duplicate a -spec from +%% rabbit_backing_queue with the only intent being to remove +%% warnings. The problem here is that we can't parameterise the BQ +%% behaviour by these two types as we would like to. We still leave +%% these here for documentation purposes. +-type ack() :: seq_id(). +-type state() :: #vqstate { + q1 :: ?QUEUE:?QUEUE(), + q2 :: ?QUEUE:?QUEUE(), + delta :: delta(), + q3 :: ?QUEUE:?QUEUE(), + q4 :: ?QUEUE:?QUEUE(), + next_seq_id :: seq_id(), + ram_pending_ack :: gb_trees:tree(), + disk_pending_ack :: gb_trees:tree(), + qi_pending_ack :: gb_trees:tree(), + index_state :: any(), + msg_store_clients :: 'undefined' | {{any(), binary()}, + {any(), binary()}}, + durable :: boolean(), + transient_threshold :: non_neg_integer(), + qi_embed_msgs_below :: non_neg_integer(), + + len :: non_neg_integer(), + bytes :: non_neg_integer(), + unacked_bytes :: non_neg_integer(), + + persistent_count :: non_neg_integer(), + persistent_bytes :: non_neg_integer(), + + target_ram_count :: non_neg_integer() | 'infinity', + ram_msg_count :: non_neg_integer(), + ram_msg_count_prev :: non_neg_integer(), + ram_ack_count_prev :: non_neg_integer(), + ram_bytes :: non_neg_integer(), + out_counter :: non_neg_integer(), + in_counter :: non_neg_integer(), + rates :: rates(), + msgs_on_disk :: gb_sets:set(), + msg_indices_on_disk :: gb_sets:set(), + unconfirmed :: gb_sets:set(), + confirmed :: gb_sets:set(), + ack_out_counter :: non_neg_integer(), + ack_in_counter :: non_neg_integer(), + disk_read_count :: non_neg_integer(), + disk_write_count :: non_neg_integer(), + + io_batch_size :: pos_integer(), + mode :: 'default' | 'lazy', + virtual_host :: rabbit_types:vhost() }. +%% Duplicated from rabbit_backing_queue +-spec ack([ack()], state()) -> {[rabbit_guid:guid()], state()}. + +%%---------------------------------------------------------------------------- +%% Public API +%%---------------------------------------------------------------------------- + +start(VHost, DurableQueues) -> + rabbit_variable_queue:start(VHost, DurableQueues). + +stop(VHost) -> + rabbit_variable_queue:stop(VHost). + +init(Queue, Recover, Callback) -> + rabbit_variable_queue:init(Queue, Recover, Callback). + +terminate(Reason, State) -> + rabbit_variable_queue:terminate(Reason, State). + +delete_and_terminate(Reason, State) -> + rabbit_variable_queue:delete_and_terminate(Reason, State). + +delete_crashed(Q) -> + rabbit_variable_queue:delete_crashed(Q). + +purge(State = #vqstate { qi_pending_ack= QPA }) -> + maybe_delay(QPA), + rabbit_variable_queue:purge(State). + +purge_acks(State) -> + rabbit_variable_queue:purge_acks(State). + +publish(Msg, MsgProps, IsDelivered, ChPid, Flow, State) -> + rabbit_variable_queue:publish(Msg, MsgProps, IsDelivered, ChPid, Flow, State). + +batch_publish(Publishes, ChPid, Flow, State) -> + rabbit_variable_queue:batch_publish(Publishes, ChPid, Flow, State). + +publish_delivered(Msg, MsgProps, ChPid, Flow, State) -> + rabbit_variable_queue:publish_delivered(Msg, MsgProps, ChPid, Flow, State). + +batch_publish_delivered(Publishes, ChPid, Flow, State) -> + rabbit_variable_queue:batch_publish_delivered(Publishes, ChPid, Flow, State). + +discard(_MsgId, _ChPid, _Flow, State) -> State. + +drain_confirmed(State) -> + rabbit_variable_queue:drain_confirmed(State). + +dropwhile(Pred, State) -> + rabbit_variable_queue:dropwhile(Pred, State). + +fetchwhile(Pred, Fun, Acc, State) -> + rabbit_variable_queue:fetchwhile(Pred, Fun, Acc, State). + +fetch(AckRequired, State) -> + rabbit_variable_queue:fetch(AckRequired, State). + +drop(AckRequired, State) -> + rabbit_variable_queue:drop(AckRequired, State). + +ack(List, State) -> + rabbit_variable_queue:ack(List, State). + +requeue(AckTags, #vqstate { qi_pending_ack = QPA } = State) -> + maybe_delay(QPA), + rabbit_variable_queue:requeue(AckTags, State). + +ackfold(MsgFun, Acc, State, AckTags) -> + rabbit_variable_queue:ackfold(MsgFun, Acc, State, AckTags). + +fold(Fun, Acc, State) -> + rabbit_variable_queue:fold(Fun, Acc, State). + +len(#vqstate { qi_pending_ack = QPA } = State) -> + maybe_delay(QPA), + rabbit_variable_queue:len(State). + +is_empty(State) -> 0 == len(State). + +depth(State) -> + rabbit_variable_queue:depth(State). + +set_ram_duration_target(DurationTarget, State) -> + rabbit_variable_queue:set_ram_duration_target(DurationTarget, State). + +ram_duration(State) -> + rabbit_variable_queue:ram_duration(State). + +needs_timeout(State) -> + rabbit_variable_queue:needs_timeout(State). + +timeout(State) -> + rabbit_variable_queue:timeout(State). + +handle_pre_hibernate(State) -> + rabbit_variable_queue:handle_pre_hibernate(State). + +handle_info(Msg, State) -> + rabbit_variable_queue:handle_info(Msg, State). + +resume(State) -> rabbit_variable_queue:resume(State). + +msg_rates(State) -> + rabbit_variable_queue:msg_rates(State). + +info(Info, State) -> + rabbit_variable_queue:info(Info, State). + +invoke(Module, Fun, State) -> rabbit_variable_queue:invoke(Module, Fun, State). + +is_duplicate(Msg, State) -> rabbit_variable_queue:is_duplicate(Msg, State). + +set_queue_mode(Mode, State) -> + rabbit_variable_queue:set_queue_mode(Mode, State). + +zip_msgs_and_acks(Msgs, AckTags, Accumulator, State) -> + rabbit_variable_queue:zip_msgs_and_acks(Msgs, AckTags, Accumulator, State). + +%% Delay +maybe_delay(QPA) -> + case is_timeout_test(gb_trees:values(QPA)) of + true -> receive + %% The queue received an EXIT message, it's probably the + %% node being stopped with "rabbitmqctl stop". Thus, abort + %% the wait and requeue the EXIT message. + {'EXIT', _, shutdown} = ExitMsg -> self() ! ExitMsg, + void + after infinity -> void + end; + _ -> void + end. + +is_timeout_test([]) -> false; +is_timeout_test([#msg_status{ + msg = #basic_message{ + content = #content{ + payload_fragments_rev = PFR}}}|Rem]) -> + case lists:member(?TIMEOUT_TEST_MSG, PFR) of + T = true -> T; + _ -> is_timeout_test(Rem) + end; +is_timeout_test([_|Rem]) -> is_timeout_test(Rem). diff --git a/deps/rabbit/test/cluster_SUITE.erl b/deps/rabbit/test/cluster_SUITE.erl new file mode 100644 index 0000000000..9df196a8ed --- /dev/null +++ b/deps/rabbit/test/cluster_SUITE.erl @@ -0,0 +1,307 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2011-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(cluster_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). +-include("include/amqqueue.hrl"). + +-compile(export_all). + +-define(TIMEOUT, 30000). + +-define(CLEANUP_QUEUE_NAME, <<"cleanup-queue">>). + +-define(CLUSTER_TESTCASES, [ + delegates_async, + delegates_sync, + queue_cleanup, + declare_on_dead_queue + ]). + +all() -> + [ + {group, cluster_tests} + ]. + +groups() -> + [ + {cluster_tests, [], [ + {from_cluster_node1, [], ?CLUSTER_TESTCASES}, + {from_cluster_node2, [], ?CLUSTER_TESTCASES} + ]} + ]. + +group(_) -> + []. + +%% ------------------------------------------------------------------- +%% Testsuite setup/teardown. +%% ------------------------------------------------------------------- + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(Group, Config) -> + case lists:member({group, Group}, all()) of + true -> + Config1 = rabbit_ct_helpers:set_config(Config, [ + {rmq_nodename_suffix, Group}, + {rmq_nodes_count, 2} + ]), + rabbit_ct_helpers:run_steps(Config1, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps() ++ [ + fun(C) -> init_per_group1(Group, C) end + ]); + false -> + rabbit_ct_helpers:run_steps(Config, [ + fun(C) -> init_per_group1(Group, C) end + ]) + end. + +init_per_group1(from_cluster_node1, Config) -> + rabbit_ct_helpers:set_config(Config, {test_direction, {0, 1}}); +init_per_group1(from_cluster_node2, Config) -> + rabbit_ct_helpers:set_config(Config, {test_direction, {1, 0}}); +init_per_group1(_, Config) -> + Config. + +end_per_group(Group, Config) -> + case lists:member({group, Group}, all()) of + true -> + rabbit_ct_helpers:run_steps(Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()); + false -> + Config + end. + +init_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_started(Config, Testcase). + +end_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_finished(Config, Testcase). + +%% --------------------------------------------------------------------------- +%% Cluster-dependent tests. +%% --------------------------------------------------------------------------- + +delegates_async(Config) -> + {I, J} = ?config(test_direction, Config), + From = rabbit_ct_broker_helpers:get_node_config(Config, I, nodename), + To = rabbit_ct_broker_helpers:get_node_config(Config, J, nodename), + rabbit_ct_broker_helpers:add_code_path_to_node(To, ?MODULE), + passed = rabbit_ct_broker_helpers:rpc(Config, + From, ?MODULE, delegates_async1, [Config, To]). + +delegates_async1(_Config, SecondaryNode) -> + Self = self(), + Sender = fun (Pid) -> Pid ! {invoked, Self} end, + + Responder = make_responder(fun ({invoked, Pid}) -> Pid ! response end), + + ok = delegate:invoke_no_result(spawn(Responder), Sender), + ok = delegate:invoke_no_result(spawn(SecondaryNode, Responder), Sender), + await_response(2), + + passed. + +delegates_sync(Config) -> + {I, J} = ?config(test_direction, Config), + From = rabbit_ct_broker_helpers:get_node_config(Config, I, nodename), + To = rabbit_ct_broker_helpers:get_node_config(Config, J, nodename), + rabbit_ct_broker_helpers:add_code_path_to_node(To, ?MODULE), + passed = rabbit_ct_broker_helpers:rpc(Config, + From, ?MODULE, delegates_sync1, [Config, To]). + +delegates_sync1(_Config, SecondaryNode) -> + Sender = fun (Pid) -> gen_server:call(Pid, invoked, infinity) end, + BadSender = fun (_Pid) -> exit(exception) end, + + Responder = make_responder(fun ({'$gen_call', From, invoked}) -> + gen_server:reply(From, response) + end), + + BadResponder = make_responder(fun ({'$gen_call', From, invoked}) -> + gen_server:reply(From, response) + end, bad_responder_died), + + response = delegate:invoke(spawn(Responder), Sender), + response = delegate:invoke(spawn(SecondaryNode, Responder), Sender), + + must_exit(fun () -> delegate:invoke(spawn(BadResponder), BadSender) end), + must_exit(fun () -> + delegate:invoke(spawn(SecondaryNode, BadResponder), BadSender) end), + + LocalGoodPids = spawn_responders(node(), Responder, 2), + RemoteGoodPids = spawn_responders(SecondaryNode, Responder, 2), + LocalBadPids = spawn_responders(node(), BadResponder, 2), + RemoteBadPids = spawn_responders(SecondaryNode, BadResponder, 2), + + {GoodRes, []} = delegate:invoke(LocalGoodPids ++ RemoteGoodPids, Sender), + true = lists:all(fun ({_, response}) -> true end, GoodRes), + GoodResPids = [Pid || {Pid, _} <- GoodRes], + + Good = lists:usort(LocalGoodPids ++ RemoteGoodPids), + Good = lists:usort(GoodResPids), + + {[], BadRes} = delegate:invoke(LocalBadPids ++ RemoteBadPids, BadSender), + true = lists:all(fun ({_, {exit, exception, _}}) -> true end, BadRes), + BadResPids = [Pid || {Pid, _} <- BadRes], + + Bad = lists:usort(LocalBadPids ++ RemoteBadPids), + Bad = lists:usort(BadResPids), + + MagicalPids = [rabbit_misc:string_to_pid(Str) || + Str <- ["<nonode@nohost.0.1.0>", "<nonode@nohost.0.2.0>"]], + {[], BadNodes} = delegate:invoke(MagicalPids, Sender), + true = lists:all( + fun ({_, {exit, {nodedown, nonode@nohost}, _Stack}}) -> true end, + BadNodes), + BadNodesPids = [Pid || {Pid, _} <- BadNodes], + + Magical = lists:usort(MagicalPids), + Magical = lists:usort(BadNodesPids), + + passed. + +queue_cleanup(Config) -> + {I, J} = ?config(test_direction, Config), + From = rabbit_ct_broker_helpers:get_node_config(Config, I, nodename), + To = rabbit_ct_broker_helpers:get_node_config(Config, J, nodename), + rabbit_ct_broker_helpers:add_code_path_to_node(To, ?MODULE), + passed = rabbit_ct_broker_helpers:rpc(Config, + From, ?MODULE, queue_cleanup1, [Config, To]). + +queue_cleanup1(_Config, _SecondaryNode) -> + {_Writer, Ch} = test_spawn(), + rabbit_channel:do(Ch, #'queue.declare'{ queue = ?CLEANUP_QUEUE_NAME }), + receive #'queue.declare_ok'{queue = ?CLEANUP_QUEUE_NAME} -> + ok + after ?TIMEOUT -> throw(failed_to_receive_queue_declare_ok) + end, + rabbit_channel:shutdown(Ch), + rabbit:stop(), + rabbit:start(), + {_Writer2, Ch2} = test_spawn(), + rabbit_channel:do(Ch2, #'queue.declare'{ passive = true, + queue = ?CLEANUP_QUEUE_NAME }), + receive + #'channel.close'{reply_code = ?NOT_FOUND} -> + ok + after ?TIMEOUT -> throw(failed_to_receive_channel_exit) + end, + rabbit_channel:shutdown(Ch2), + passed. + +declare_on_dead_queue(Config) -> + {I, J} = ?config(test_direction, Config), + From = rabbit_ct_broker_helpers:get_node_config(Config, I, nodename), + To = rabbit_ct_broker_helpers:get_node_config(Config, J, nodename), + rabbit_ct_broker_helpers:add_code_path_to_node(To, ?MODULE), + passed = rabbit_ct_broker_helpers:rpc(Config, + From, ?MODULE, declare_on_dead_queue1, [Config, To]). + +declare_on_dead_queue1(_Config, SecondaryNode) -> + QueueName = rabbit_misc:r(<<"/">>, queue, ?CLEANUP_QUEUE_NAME), + Self = self(), + Pid = spawn(SecondaryNode, + fun () -> + {new, Q} = rabbit_amqqueue:declare(QueueName, false, false, [], none, <<"acting-user">>), + QueueName = ?amqqueue_field_name(Q), + QPid = ?amqqueue_field_pid(Q), + exit(QPid, kill), + Self ! {self(), killed, QPid} + end), + receive + {Pid, killed, OldPid} -> + Q = dead_queue_loop(QueueName, OldPid), + {ok, 0} = rabbit_amqqueue:delete(Q, false, false, <<"acting-user">>), + passed + after ?TIMEOUT -> throw(failed_to_create_and_kill_queue) + end. + + +make_responder(FMsg) -> make_responder(FMsg, timeout). +make_responder(FMsg, Throw) -> + fun () -> + receive Msg -> FMsg(Msg) + after ?TIMEOUT -> throw(Throw) + end + end. + +spawn_responders(Node, Responder, Count) -> + [spawn(Node, Responder) || _ <- lists:seq(1, Count)]. + +await_response(0) -> + ok; +await_response(Count) -> + receive + response -> ok, + await_response(Count - 1) + after ?TIMEOUT -> throw(timeout) + end. + +must_exit(Fun) -> + try + Fun(), + throw(exit_not_thrown) + catch + exit:_ -> ok + end. + +dead_queue_loop(QueueName, OldPid) -> + {existing, Q} = rabbit_amqqueue:declare(QueueName, false, false, [], none, <<"acting-user">>), + QPid = ?amqqueue_field_pid(Q), + case QPid of + OldPid -> timer:sleep(25), + dead_queue_loop(QueueName, OldPid); + _ -> true = rabbit_misc:is_process_alive(QPid), + Q + end. + + +test_spawn() -> + {Writer, _Limiter, Ch} = rabbit_ct_broker_helpers:test_channel(), + ok = rabbit_channel:do(Ch, #'channel.open'{}), + receive #'channel.open_ok'{} -> ok + after ?TIMEOUT -> throw(failed_to_receive_channel_open_ok) + end, + {Writer, Ch}. + +test_spawn(Node) -> + rpc:call(Node, ?MODULE, test_spawn_remote, []). + +%% Spawn an arbitrary long lived process, so we don't end up linking +%% the channel to the short-lived process (RPC, here) spun up by the +%% RPC server. +test_spawn_remote() -> + RPC = self(), + spawn(fun () -> + {Writer, Ch} = test_spawn(), + RPC ! {Writer, Ch}, + link(Ch), + receive + _ -> ok + end + end), + receive Res -> Res + after ?TIMEOUT -> throw(failed_to_receive_result) + end. + +queue_name(Config, Name) -> + Name1 = iolist_to_binary(rabbit_ct_helpers:config_to_testcase_name(Config, Name)), + queue_name(Name1). + +queue_name(Name) -> + rabbit_misc:r(<<"/">>, queue, Name). diff --git a/deps/rabbit/test/cluster_rename_SUITE.erl b/deps/rabbit/test/cluster_rename_SUITE.erl new file mode 100644 index 0000000000..cdf02c9643 --- /dev/null +++ b/deps/rabbit/test/cluster_rename_SUITE.erl @@ -0,0 +1,301 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(cluster_rename_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). + +-compile(export_all). + +all() -> + [ + {group, cluster_size_2}, + {group, cluster_size_3} + ]. + +groups() -> + [ + {cluster_size_2, [], [ + % XXX post_change_nodename, + abortive_rename, + rename_fail, + rename_twice_fail + ]}, + {cluster_size_3, [], [ + rename_cluster_one_by_one, + rename_cluster_big_bang, + partial_one_by_one, + partial_big_bang + ]} + ]. + +suite() -> + [ + %% If a test hangs, no need to wait for 30 minutes. + {timetrap, {minutes, 15}} + ]. + +%% ------------------------------------------------------------------- +%% Testsuite setup/teardown. +%% ------------------------------------------------------------------- + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(cluster_size_2, Config) -> + rabbit_ct_helpers:set_config(Config, [ + {rmq_nodes_count, 2} %% Replaced with a list of node names later. + ]); +init_per_group(cluster_size_3, Config) -> + rabbit_ct_helpers:set_config(Config, [ + {rmq_nodes_count, 3} %% Replaced with a list of node names later. + ]). + +end_per_group(_, Config) -> + Config. + +init_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_started(Config, Testcase), + ClusterSize = ?config(rmq_nodes_count, Config), + Nodenames = [ + list_to_atom(rabbit_misc:format("~s-~b", [Testcase, I])) + || I <- lists:seq(1, ClusterSize) + ], + Config1 = rabbit_ct_helpers:set_config(Config, [ + {rmq_nodes_count, Nodenames}, + {rmq_nodes_clustered, true} + ]), + rabbit_ct_helpers:run_steps(Config1, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()). + +end_per_testcase(Testcase, Config) -> + Config1 = case rabbit_ct_helpers:get_config(Config, save_config) of + undefined -> Config; + C -> C + end, + Config2 = rabbit_ct_helpers:run_steps(Config1, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()), + rabbit_ct_helpers:testcase_finished(Config2, Testcase). + +%% ------------------------------------------------------------------- +%% Testcases. +%% ------------------------------------------------------------------- + +%% Rolling rename of a cluster, each node should do a secondary rename. +rename_cluster_one_by_one(Config) -> + [Node1, Node2, Node3] = rabbit_ct_broker_helpers:get_node_configs( + Config, nodename), + publish_all(Config, + [{Node1, <<"1">>}, {Node2, <<"2">>}, {Node3, <<"3">>}]), + + Config1 = stop_rename_start(Config, Node1, [Node1, jessica]), + Config2 = stop_rename_start(Config1, Node2, [Node2, hazel]), + Config3 = stop_rename_start(Config2, Node3, [Node3, flopsy]), + + [Jessica, Hazel, Flopsy] = rabbit_ct_broker_helpers:get_node_configs( + Config3, nodename), + consume_all(Config3, + [{Jessica, <<"1">>}, {Hazel, <<"2">>}, {Flopsy, <<"3">>}]), + {save_config, Config3}. + +%% Big bang rename of a cluster, Node1 should do a primary rename. +rename_cluster_big_bang(Config) -> + [Node1, Node2, Node3] = rabbit_ct_broker_helpers:get_node_configs(Config, + nodename), + publish_all(Config, + [{Node1, <<"1">>}, {Node2, <<"2">>}, {Node3, <<"3">>}]), + + ok = rabbit_ct_broker_helpers:stop_node(Config, Node3), + ok = rabbit_ct_broker_helpers:stop_node(Config, Node2), + ok = rabbit_ct_broker_helpers:stop_node(Config, Node1), + + Map = [Node1, jessica, Node2, hazel, Node3, flopsy], + Config1 = rename_node(Config, Node1, Map), + Config2 = rename_node(Config1, Node2, Map), + Config3 = rename_node(Config2, Node3, Map), + + [Jessica, Hazel, Flopsy] = rabbit_ct_broker_helpers:get_node_configs( + Config3, nodename), + ok = rabbit_ct_broker_helpers:start_node(Config3, Jessica), + ok = rabbit_ct_broker_helpers:start_node(Config3, Hazel), + ok = rabbit_ct_broker_helpers:start_node(Config3, Flopsy), + + consume_all(Config3, + [{Jessica, <<"1">>}, {Hazel, <<"2">>}, {Flopsy, <<"3">>}]), + {save_config, Config3}. + +%% Here we test that Node1 copes with things being renamed around it. +partial_one_by_one(Config) -> + [Node1, Node2, Node3] = rabbit_ct_broker_helpers:get_node_configs(Config, + nodename), + publish_all(Config, + [{Node1, <<"1">>}, {Node2, <<"2">>}, {Node3, <<"3">>}]), + + Config1 = stop_rename_start(Config, Node1, [Node1, jessica]), + Config2 = stop_rename_start(Config1, Node2, [Node2, hazel]), + + [Jessica, Hazel, Node3] = rabbit_ct_broker_helpers:get_node_configs( + Config2, nodename), + consume_all(Config2, + [{Jessica, <<"1">>}, {Hazel, <<"2">>}, {Node3, <<"3">>}]), + {save_config, Config2}. + +%% Here we test that Node1 copes with things being renamed around it. +partial_big_bang(Config) -> + [Node1, Node2, Node3] = rabbit_ct_broker_helpers:get_node_configs(Config, + nodename), + publish_all(Config, + [{Node1, <<"1">>}, {Node2, <<"2">>}, {Node3, <<"3">>}]), + + ok = rabbit_ct_broker_helpers:stop_node(Config, Node3), + ok = rabbit_ct_broker_helpers:stop_node(Config, Node2), + ok = rabbit_ct_broker_helpers:stop_node(Config, Node1), + + Map = [Node2, hazel, Node3, flopsy], + Config1 = rename_node(Config, Node2, Map), + Config2 = rename_node(Config1, Node3, Map), + + [Node1, Hazel, Flopsy] = rabbit_ct_broker_helpers:get_node_configs(Config2, + nodename), + ok = rabbit_ct_broker_helpers:start_node(Config2, Node1), + ok = rabbit_ct_broker_helpers:start_node(Config2, Hazel), + ok = rabbit_ct_broker_helpers:start_node(Config2, Flopsy), + + consume_all(Config2, + [{Node1, <<"1">>}, {Hazel, <<"2">>}, {Flopsy, <<"3">>}]), + {save_config, Config2}. + +% XXX %% We should be able to specify the -n parameter on ctl with either +% XXX %% the before or after name for the local node (since in real cases +% XXX %% one might want to invoke the command before or after the hostname +% XXX %% has changed) - usually we test before so here we test after. +% XXX post_change_nodename([Node1, _Bigwig]) -> +% XXX publish(Node1, <<"Node1">>), +% XXX +% XXX Bugs1 = rabbit_test_configs:stop_node(Node1), +% XXX Bugs2 = [{nodename, jessica} | proplists:delete(nodename, Bugs1)], +% XXX Jessica0 = rename_node(Bugs2, jessica, [Node1, jessica]), +% XXX Jessica = rabbit_test_configs:start_node(Jessica0), +% XXX +% XXX consume(Jessica, <<"Node1">>), +% XXX stop_all([Jessica]), +% XXX ok. + +%% If we invoke rename but the node name does not actually change, we +%% should roll back. +abortive_rename(Config) -> + Node1 = rabbit_ct_broker_helpers:get_node_config(Config, 0, nodename), + publish(Config, Node1, <<"Node1">>), + + ok = rabbit_ct_broker_helpers:stop_node(Config, Node1), + _Config1 = rename_node(Config, Node1, [Node1, jessica]), + ok = rabbit_ct_broker_helpers:start_node(Config, Node1), + + consume(Config, Node1, <<"Node1">>), + ok. + +%% And test some ways the command can fail. +rename_fail(Config) -> + [Node1, Node2] = rabbit_ct_broker_helpers:get_node_configs(Config, + nodename), + ok = rabbit_ct_broker_helpers:stop_node(Config, Node1), + %% Rename from a node that does not exist + ok = rename_node_fail(Config, Node1, [bugzilla, jessica]), + %% Rename to a node which does + ok = rename_node_fail(Config, Node1, [Node1, Node2]), + %% Rename two nodes to the same thing + ok = rename_node_fail(Config, Node1, [Node1, jessica, Node2, jessica]), + %% Rename while impersonating a node not in the cluster + Config1 = rabbit_ct_broker_helpers:set_node_config(Config, Node1, + {nodename, 'rabbit@localhost'}), + ok = rename_node_fail(Config1, Node1, [Node1, jessica]), + ok. + +rename_twice_fail(Config) -> + Node1 = rabbit_ct_broker_helpers:get_node_config(Config, 0, nodename), + ok = rabbit_ct_broker_helpers:stop_node(Config, Node1), + Config1 = rename_node(Config, Node1, [Node1, indecisive]), + ok = rename_node_fail(Config, Node1, [indecisive, jessica]), + {save_config, Config1}. + +%% ---------------------------------------------------------------------------- + +stop_rename_start(Config, Nodename, Map) -> + ok = rabbit_ct_broker_helpers:stop_node(Config, Nodename), + Config1 = rename_node(Config, Nodename, Map), + ok = rabbit_ct_broker_helpers:start_node(Config1, Nodename), + Config1. + +rename_node(Config, Nodename, Map) -> + {ok, Config1} = do_rename_node(Config, Nodename, Map), + Config1. + +rename_node_fail(Config, Nodename, Map) -> + {error, _, _} = do_rename_node(Config, Nodename, Map), + ok. + +do_rename_node(Config, Nodename, Map) -> + Map1 = [ + begin + NStr = atom_to_list(N), + case lists:member($@, NStr) of + true -> N; + false -> rabbit_nodes:make({NStr, "localhost"}) + end + end + || N <- Map + ], + Ret = rabbit_ct_broker_helpers:rabbitmqctl(Config, Nodename, + ["rename_cluster_node" | Map1], 120000), + case Ret of + {ok, _} -> + Config1 = update_config_after_rename(Config, Map1), + {ok, Config1}; + {error, _, _} = Error -> + Error + end. + +update_config_after_rename(Config, [Old, New | Rest]) -> + Config1 = rabbit_ct_broker_helpers:set_node_config(Config, Old, + {nodename, New}), + update_config_after_rename(Config1, Rest); +update_config_after_rename(Config, []) -> + Config. + +publish(Config, Node, Q) -> + Ch = rabbit_ct_client_helpers:open_channel(Config, Node), + amqp_channel:call(Ch, #'confirm.select'{}), + amqp_channel:call(Ch, #'queue.declare'{queue = Q, durable = true}), + amqp_channel:cast(Ch, #'basic.publish'{routing_key = Q}, + #amqp_msg{props = #'P_basic'{delivery_mode = 2}, + payload = Q}), + amqp_channel:wait_for_confirms(Ch), + rabbit_ct_client_helpers:close_channels_and_connection(Config, Node). + +consume(Config, Node, Q) -> + Ch = rabbit_ct_client_helpers:open_channel(Config, Node), + amqp_channel:call(Ch, #'queue.declare'{queue = Q, durable = true}), + {#'basic.get_ok'{}, #amqp_msg{payload = Q}} = + amqp_channel:call(Ch, #'basic.get'{queue = Q}), + rabbit_ct_client_helpers:close_channels_and_connection(Config, Node). + + +publish_all(Config, Nodes) -> + [publish(Config, Node, Key) || {Node, Key} <- Nodes]. + +consume_all(Config, Nodes) -> + [consume(Config, Node, Key) || {Node, Key} <- Nodes]. + +set_node(Nodename, Cfg) -> + [{nodename, Nodename} | proplists:delete(nodename, Cfg)]. diff --git a/deps/rabbit/test/clustering_management_SUITE.erl b/deps/rabbit/test/clustering_management_SUITE.erl new file mode 100644 index 0000000000..550a30b511 --- /dev/null +++ b/deps/rabbit/test/clustering_management_SUITE.erl @@ -0,0 +1,861 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(clustering_management_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-compile(export_all). + +-define(LOOP_RECURSION_DELAY, 100). + +all() -> + [ + {group, unclustered_2_nodes}, + {group, unclustered_3_nodes}, + {group, clustered_2_nodes}, + {group, clustered_4_nodes} + ]. + +groups() -> + [ + {unclustered_2_nodes, [], [ + {cluster_size_2, [], [ + classic_config_discovery_node_list + ]} + ]}, + {unclustered_3_nodes, [], [ + {cluster_size_3, [], [ + join_and_part_cluster, + join_cluster_bad_operations, + join_to_start_interval, + forget_cluster_node, + change_cluster_node_type, + change_cluster_when_node_offline, + update_cluster_nodes, + force_reset_node + ]} + ]}, + {clustered_2_nodes, [], [ + {cluster_size_2, [], [ + forget_removes_things, + reset_removes_things, + forget_offline_removes_things, + force_boot, + status_with_alarm, + pid_file_and_await_node_startup, + await_running_count, + start_with_invalid_schema_in_path, + persistent_cluster_id + ]} + ]}, + {clustered_4_nodes, [], [ + {cluster_size_4, [], [ + forget_promotes_offline_follower + ]} + ]} + ]. + +suite() -> + [ + %% If a test hangs, no need to wait for 30 minutes. + {timetrap, {minutes, 15}} + ]. + +%% ------------------------------------------------------------------- +%% Testsuite setup/teardown. +%% ------------------------------------------------------------------- + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + Config1 = rabbit_ct_helpers:merge_app_env( + Config, {rabbit, [ + {mnesia_table_loading_retry_limit, 2}, + {mnesia_table_loading_retry_timeout,1000} + ]}), + rabbit_ct_helpers:run_setup_steps(Config1). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(unclustered_2_nodes, Config) -> + rabbit_ct_helpers:set_config(Config, [{rmq_nodes_clustered, false}]); +init_per_group(unclustered_3_nodes, Config) -> + rabbit_ct_helpers:set_config(Config, [{rmq_nodes_clustered, false}]); +init_per_group(clustered_2_nodes, Config) -> + rabbit_ct_helpers:set_config(Config, [{rmq_nodes_clustered, true}]); +init_per_group(clustered_4_nodes, Config) -> + rabbit_ct_helpers:set_config(Config, [{rmq_nodes_clustered, true}]); +init_per_group(cluster_size_2, Config) -> + rabbit_ct_helpers:set_config(Config, [{rmq_nodes_count, 2}]); +init_per_group(cluster_size_3, Config) -> + rabbit_ct_helpers:set_config(Config, [{rmq_nodes_count, 3}]); +init_per_group(cluster_size_4, Config) -> + rabbit_ct_helpers:set_config(Config, [{rmq_nodes_count, 4}]). + +end_per_group(_, Config) -> + Config. + +init_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_started(Config, Testcase), + ClusterSize = ?config(rmq_nodes_count, Config), + TestNumber = rabbit_ct_helpers:testcase_number(Config, ?MODULE, Testcase), + Config1 = rabbit_ct_helpers:set_config(Config, [ + {rmq_nodename_suffix, Testcase}, + {tcp_ports_base, {skip_n_nodes, TestNumber * ClusterSize}}, + {keep_pid_file_on_exit, true} + ]), + rabbit_ct_helpers:run_steps(Config1, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()). + +end_per_testcase(Testcase, Config) -> + Config1 = rabbit_ct_helpers:run_steps(Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()), + rabbit_ct_helpers:testcase_finished(Config1, Testcase). + +%% ------------------------------------------------------------------- +%% Test cases +%% ------------------------------------------------------------------- + +start_with_invalid_schema_in_path(Config) -> + [Rabbit, Hare] = cluster_members(Config), + stop_app(Rabbit), + stop_app(Hare), + + create_bad_schema(Rabbit, Hare, Config), + + start_app(Hare), + case start_app(Rabbit) of + ok -> ok; + ErrRabbit -> error({unable_to_start_with_bad_schema_in_work_dir, ErrRabbit}) + end. + +persistent_cluster_id(Config) -> + case os:getenv("SECONDARY_UMBRELLA") of + false -> + [Rabbit, Hare] = cluster_members(Config), + ClusterIDA1 = rpc:call(Rabbit, rabbit_nodes, persistent_cluster_id, []), + ClusterIDB1 = rpc:call(Hare, rabbit_nodes, persistent_cluster_id, []), + ?assertEqual(ClusterIDA1, ClusterIDB1), + + rabbit_ct_broker_helpers:restart_node(Config, Rabbit), + ClusterIDA2 = rpc:call(Rabbit, rabbit_nodes, persistent_cluster_id, []), + rabbit_ct_broker_helpers:restart_node(Config, Hare), + ClusterIDB2 = rpc:call(Hare, rabbit_nodes, persistent_cluster_id, []), + ?assertEqual(ClusterIDA1, ClusterIDA2), + ?assertEqual(ClusterIDA2, ClusterIDB2); + _ -> + %% skip the test in mixed version mode + {skip, "Should not run in mixed version environments"} + end. + +create_bad_schema(Rabbit, Hare, Config) -> + {ok, RabbitMnesiaDir} = rpc:call(Rabbit, application, get_env, [mnesia, dir]), + {ok, HareMnesiaDir} = rpc:call(Hare, application, get_env, [mnesia, dir]), + %% Make sure we don't use the current dir: + PrivDir = ?config(priv_dir, Config), + ct:pal("Priv dir ~p~n", [PrivDir]), + ok = filelib:ensure_dir(filename:join(PrivDir, "file")), + + ok = rpc:call(Rabbit, file, set_cwd, [PrivDir]), + ok = rpc:call(Hare, file, set_cwd, [PrivDir]), + + ok = rpc:call(Rabbit, application, unset_env, [mnesia, dir]), + ok = rpc:call(Hare, application, unset_env, [mnesia, dir]), + ok = rpc:call(Rabbit, mnesia, create_schema, [[Rabbit, Hare]]), + ok = rpc:call(Rabbit, mnesia, start, []), + {atomic,ok} = rpc:call(Rabbit, mnesia, create_table, + [rabbit_queue, [{ram_copies, [Rabbit, Hare]}]]), + stopped = rpc:call(Rabbit, mnesia, stop, []), + ok = rpc:call(Rabbit, application, set_env, [mnesia, dir, RabbitMnesiaDir]), + ok = rpc:call(Hare, application, set_env, [mnesia, dir, HareMnesiaDir]). + +join_and_part_cluster(Config) -> + [Rabbit, Hare, Bunny] = cluster_members(Config), + assert_not_clustered(Rabbit), + assert_not_clustered(Hare), + assert_not_clustered(Bunny), + + stop_join_start(Rabbit, Bunny), + assert_clustered([Rabbit, Bunny]), + + stop_join_start(Hare, Bunny, true), + assert_cluster_status( + {[Bunny, Hare, Rabbit], [Bunny, Rabbit], [Bunny, Hare, Rabbit]}, + [Rabbit, Hare, Bunny]), + + %% Allow clustering with already clustered node + ok = stop_app(Rabbit), + {ok, <<"The node is already a member of this cluster">>} = + join_cluster(Rabbit, Hare), + ok = start_app(Rabbit), + + stop_reset_start(Rabbit), + assert_not_clustered(Rabbit), + assert_cluster_status({[Bunny, Hare], [Bunny], [Bunny, Hare]}, + [Hare, Bunny]), + + stop_reset_start(Hare), + assert_not_clustered(Hare), + assert_not_clustered(Bunny). + +join_cluster_bad_operations(Config) -> + [Rabbit, Hare, Bunny] = cluster_members(Config), + + UsePrelaunch = rabbit_ct_broker_helpers:rpc( + Config, Hare, + erlang, function_exported, + [rabbit_prelaunch, get_context, 0]), + + %% Nonexistent node + ok = stop_app(Rabbit), + assert_failure(fun () -> join_cluster(Rabbit, non@existent) end), + ok = start_app(Rabbit), + assert_not_clustered(Rabbit), + + %% Trying to cluster with mnesia running + assert_failure(fun () -> join_cluster(Rabbit, Bunny) end), + assert_not_clustered(Rabbit), + + %% Trying to cluster the node with itself + ok = stop_app(Rabbit), + assert_failure(fun () -> join_cluster(Rabbit, Rabbit) end), + ok = start_app(Rabbit), + assert_not_clustered(Rabbit), + + %% Do not let the node leave the cluster or reset if it's the only + %% ram node + stop_join_start(Hare, Rabbit, true), + assert_cluster_status({[Rabbit, Hare], [Rabbit], [Rabbit, Hare]}, + [Rabbit, Hare]), + ok = stop_app(Hare), + assert_failure(fun () -> join_cluster(Rabbit, Bunny) end), + assert_failure(fun () -> reset(Rabbit) end), + ok = start_app(Hare), + assert_cluster_status({[Rabbit, Hare], [Rabbit], [Rabbit, Hare]}, + [Rabbit, Hare]), + + %% Cannot start RAM-only node first + ok = stop_app(Rabbit), + ok = stop_app(Hare), + assert_failure(fun () -> start_app(Hare) end), + ok = start_app(Rabbit), + case UsePrelaunch of + true -> + ok = start_app(Hare); + false -> + %% The Erlang VM has stopped after previous rabbit app failure + ok = rabbit_ct_broker_helpers:start_node(Config, Hare) + end, + ok. + +%% This tests that the nodes in the cluster are notified immediately of a node +%% join, and not just after the app is started. +join_to_start_interval(Config) -> + [Rabbit, Hare, _Bunny] = cluster_members(Config), + + ok = stop_app(Rabbit), + ok = join_cluster(Rabbit, Hare), + assert_cluster_status({[Rabbit, Hare], [Rabbit, Hare], [Hare]}, + [Rabbit, Hare]), + ok = start_app(Rabbit), + assert_clustered([Rabbit, Hare]). + +forget_cluster_node(Config) -> + [Rabbit, Hare, Bunny] = cluster_members(Config), + + %% Trying to remove a node not in the cluster should fail + assert_failure(fun () -> forget_cluster_node(Hare, Rabbit) end), + + stop_join_start(Rabbit, Hare), + assert_clustered([Rabbit, Hare]), + + %% Trying to remove an online node should fail + assert_failure(fun () -> forget_cluster_node(Hare, Rabbit) end), + + ok = stop_app(Rabbit), + %% We're passing the --offline flag, but Hare is online + assert_failure(fun () -> forget_cluster_node(Hare, Rabbit, true) end), + %% Removing some nonexistent node will fail + assert_failure(fun () -> forget_cluster_node(Hare, non@existent) end), + ok = forget_cluster_node(Hare, Rabbit), + assert_not_clustered(Hare), + assert_cluster_status({[Rabbit, Hare], [Rabbit, Hare], [Hare]}, + [Rabbit]), + + %% Now we can't start Rabbit since it thinks that it's still in the cluster + %% with Hare, while Hare disagrees. + assert_failure(fun () -> start_app(Rabbit) end), + + ok = reset(Rabbit), + ok = start_app(Rabbit), + assert_not_clustered(Rabbit), + + %% Now we remove Rabbit from an offline node. + stop_join_start(Bunny, Hare), + stop_join_start(Rabbit, Hare), + assert_clustered([Rabbit, Hare, Bunny]), + ok = stop_app(Hare), + ok = stop_app(Rabbit), + ok = stop_app(Bunny), + %% This is fine but we need the flag + assert_failure(fun () -> forget_cluster_node(Hare, Bunny) end), + %% Also fails because hare node is still running + assert_failure(fun () -> forget_cluster_node(Hare, Bunny, true) end), + %% But this works + ok = rabbit_ct_broker_helpers:stop_node(Config, Hare), + {ok, _} = rabbit_ct_broker_helpers:rabbitmqctl(Config, Hare, + ["forget_cluster_node", "--offline", Bunny]), + ok = rabbit_ct_broker_helpers:start_node(Config, Hare), + ok = start_app(Rabbit), + %% Bunny still thinks its clustered with Rabbit and Hare + assert_failure(fun () -> start_app(Bunny) end), + ok = reset(Bunny), + ok = start_app(Bunny), + assert_not_clustered(Bunny), + assert_clustered([Rabbit, Hare]). + +forget_removes_things(Config) -> + test_removes_things(Config, fun (R, H) -> ok = forget_cluster_node(H, R) end). + +reset_removes_things(Config) -> + test_removes_things(Config, fun (R, _H) -> ok = reset(R) end). + +test_removes_things(Config, LoseRabbit) -> + Unmirrored = <<"unmirrored-queue">>, + [Rabbit, Hare] = cluster_members(Config), + RCh = rabbit_ct_client_helpers:open_channel(Config, Rabbit), + declare(RCh, Unmirrored), + ok = stop_app(Rabbit), + + HCh = rabbit_ct_client_helpers:open_channel(Config, Hare), + {'EXIT',{{shutdown,{server_initiated_close,404,_}}, _}} = + (catch declare(HCh, Unmirrored)), + + ok = LoseRabbit(Rabbit, Hare), + HCh2 = rabbit_ct_client_helpers:open_channel(Config, Hare), + declare(HCh2, Unmirrored), + ok. + +forget_offline_removes_things(Config) -> + [Rabbit, Hare] = rabbit_ct_broker_helpers:get_node_configs(Config, + nodename), + Unmirrored = <<"unmirrored-queue">>, + X = <<"X">>, + RCh = rabbit_ct_client_helpers:open_channel(Config, Rabbit), + declare(RCh, Unmirrored), + + amqp_channel:call(RCh, #'exchange.declare'{durable = true, + exchange = X, + auto_delete = true}), + amqp_channel:call(RCh, #'queue.bind'{queue = Unmirrored, + exchange = X}), + ok = rabbit_ct_broker_helpers:stop_broker(Config, Rabbit), + + HCh = rabbit_ct_client_helpers:open_channel(Config, Hare), + {'EXIT',{{shutdown,{server_initiated_close,404,_}}, _}} = + (catch declare(HCh, Unmirrored)), + + ok = rabbit_ct_broker_helpers:stop_node(Config, Hare), + ok = rabbit_ct_broker_helpers:stop_node(Config, Rabbit), + {ok, _} = rabbit_ct_broker_helpers:rabbitmqctl(Config, Hare, + ["forget_cluster_node", "--offline", Rabbit]), + ok = rabbit_ct_broker_helpers:start_node(Config, Hare), + + HCh2 = rabbit_ct_client_helpers:open_channel(Config, Hare), + declare(HCh2, Unmirrored), + {'EXIT',{{shutdown,{server_initiated_close,404,_}}, _}} = + (catch amqp_channel:call(HCh2,#'exchange.declare'{durable = true, + exchange = X, + auto_delete = true, + passive = true})), + ok. + +forget_promotes_offline_follower(Config) -> + [A, B, C, D] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + ACh = rabbit_ct_client_helpers:open_channel(Config, A), + QName = <<"mirrored-queue">>, + declare(ACh, QName), + set_ha_policy(Config, QName, A, [B, C]), + set_ha_policy(Config, QName, A, [C, D]), %% Test add and remove from recoverable_mirrors + + %% Publish and confirm + amqp_channel:call(ACh, #'confirm.select'{}), + amqp_channel:cast(ACh, #'basic.publish'{routing_key = QName}, + #amqp_msg{props = #'P_basic'{delivery_mode = 2}}), + amqp_channel:wait_for_confirms(ACh), + + %% We kill nodes rather than stop them in order to make sure + %% that we aren't dependent on anything that happens as they shut + %% down (see bug 26467). + ok = rabbit_ct_broker_helpers:kill_node(Config, D), + ok = rabbit_ct_broker_helpers:kill_node(Config, C), + ok = rabbit_ct_broker_helpers:kill_node(Config, B), + ok = rabbit_ct_broker_helpers:kill_node(Config, A), + + {ok, _} = rabbit_ct_broker_helpers:rabbitmqctl(Config, C, + ["force_boot"]), + + ok = rabbit_ct_broker_helpers:start_node(Config, C), + + %% We should now have the following dramatis personae: + %% A - down, master + %% B - down, used to be mirror, no longer is, never had the message + %% C - running, should be mirror, but has wiped the message on restart + %% D - down, recoverable mirror, contains message + %% + %% So forgetting A should offline-promote the queue to D, keeping + %% the message. + + {ok, _} = rabbit_ct_broker_helpers:rabbitmqctl(Config, C, + ["forget_cluster_node", A]), + + ok = rabbit_ct_broker_helpers:start_node(Config, D), + DCh2 = rabbit_ct_client_helpers:open_channel(Config, D), + #'queue.declare_ok'{message_count = 1} = declare(DCh2, QName), + ok. + +set_ha_policy(Config, QName, Master, Slaves) -> + Nodes = [list_to_binary(atom_to_list(N)) || N <- [Master | Slaves]], + HaPolicy = {<<"nodes">>, Nodes}, + rabbit_ct_broker_helpers:set_ha_policy(Config, Master, QName, HaPolicy), + await_followers(QName, Master, Slaves). + +await_followers(QName, Master, Slaves) -> + await_followers_0(QName, Master, Slaves, 10). + +await_followers_0(QName, Master, Slaves0, Tries) -> + {ok, Queue} = await_followers_lookup_queue(QName, Master), + SPids = amqqueue:get_slave_pids(Queue), + ActMaster = amqqueue:qnode(Queue), + ActSlaves = lists:usort([node(P) || P <- SPids]), + Slaves1 = lists:usort(Slaves0), + await_followers_1(QName, ActMaster, ActSlaves, Master, Slaves1, Tries). + +await_followers_1(QName, _ActMaster, _ActSlaves, _Master, _Slaves, 0) -> + error({timeout_waiting_for_followers, QName}); +await_followers_1(QName, ActMaster, ActSlaves, Master, Slaves, Tries) -> + case {Master, Slaves} of + {ActMaster, ActSlaves} -> + ok; + _ -> + timer:sleep(250), + await_followers_0(QName, Master, Slaves, Tries - 1) + end. + +await_followers_lookup_queue(QName, Master) -> + await_followers_lookup_queue(QName, Master, 10). + +await_followers_lookup_queue(QName, _Master, 0) -> + error({timeout_looking_up_queue, QName}); +await_followers_lookup_queue(QName, Master, Tries) -> + RpcArgs = [rabbit_misc:r(<<"/">>, queue, QName)], + case rpc:call(Master, rabbit_amqqueue, lookup, RpcArgs) of + {error, not_found} -> + timer:sleep(250), + await_followers_lookup_queue(QName, Master, Tries - 1); + {ok, Q} -> + {ok, Q} + end. + +force_boot(Config) -> + [Rabbit, Hare] = rabbit_ct_broker_helpers:get_node_configs(Config, + nodename), + {error, _, _} = rabbit_ct_broker_helpers:rabbitmqctl(Config, Rabbit, + ["force_boot"]), + ok = rabbit_ct_broker_helpers:stop_node(Config, Rabbit), + ok = rabbit_ct_broker_helpers:stop_node(Config, Hare), + {error, _} = rabbit_ct_broker_helpers:start_node(Config, Rabbit), + {ok, _} = rabbit_ct_broker_helpers:rabbitmqctl(Config, Rabbit, + ["force_boot"]), + ok = rabbit_ct_broker_helpers:start_node(Config, Rabbit), + ok. + +change_cluster_node_type(Config) -> + [Rabbit, Hare, _Bunny] = cluster_members(Config), + + %% Trying to change the node to the ram type when not clustered should always fail + ok = stop_app(Rabbit), + assert_failure(fun () -> change_cluster_node_type(Rabbit, ram) end), + ok = start_app(Rabbit), + + ok = stop_app(Rabbit), + join_cluster(Rabbit, Hare), + assert_cluster_status({[Rabbit, Hare], [Rabbit, Hare], [Hare]}, + [Rabbit, Hare]), + change_cluster_node_type(Rabbit, ram), + assert_cluster_status({[Rabbit, Hare], [Hare], [Hare]}, + [Rabbit, Hare]), + change_cluster_node_type(Rabbit, disc), + + assert_cluster_status({[Rabbit, Hare], [Rabbit, Hare], [Hare]}, + [Rabbit, Hare]), + change_cluster_node_type(Rabbit, ram), + ok = start_app(Rabbit), + assert_cluster_status({[Rabbit, Hare], [Hare], [Hare, Rabbit]}, + [Rabbit, Hare]), + + %% Changing to ram when you're the only ram node should fail + ok = stop_app(Hare), + assert_failure(fun () -> change_cluster_node_type(Hare, ram) end), + ok = start_app(Hare). + +change_cluster_when_node_offline(Config) -> + [Rabbit, Hare, Bunny] = cluster_members(Config), + + %% Cluster the three notes + stop_join_start(Rabbit, Hare), + assert_clustered([Rabbit, Hare]), + + stop_join_start(Bunny, Hare), + assert_clustered([Rabbit, Hare, Bunny]), + + %% Bring down Rabbit, and remove Bunny from the cluster while + %% Rabbit is offline + ok = stop_app(Rabbit), + ok = stop_app(Bunny), + ok = reset(Bunny), + assert_cluster_status({[Bunny], [Bunny], []}, [Bunny]), + assert_cluster_status({[Rabbit, Hare], [Rabbit, Hare], [Hare]}, [Hare]), + assert_cluster_status( + {[Rabbit, Hare, Bunny], [Rabbit, Hare, Bunny], [Hare, Bunny]}, [Rabbit]), + + %% Bring Rabbit back up + ok = start_app(Rabbit), + assert_clustered([Rabbit, Hare]), + ok = start_app(Bunny), + assert_not_clustered(Bunny), + + %% Now the same, but Rabbit is a RAM node, and we bring up Bunny + %% before + ok = stop_app(Rabbit), + ok = change_cluster_node_type(Rabbit, ram), + ok = start_app(Rabbit), + stop_join_start(Bunny, Hare), + assert_cluster_status( + {[Rabbit, Hare, Bunny], [Hare, Bunny], [Rabbit, Hare, Bunny]}, + [Rabbit, Hare, Bunny]), + ok = stop_app(Rabbit), + ok = stop_app(Bunny), + ok = reset(Bunny), + ok = start_app(Bunny), + assert_not_clustered(Bunny), + assert_cluster_status({[Rabbit, Hare], [Hare], [Hare]}, [Hare]), + assert_cluster_status( + {[Rabbit, Hare, Bunny], [Hare, Bunny], [Hare, Bunny]}, + [Rabbit]), + ok = start_app(Rabbit), + assert_cluster_status({[Rabbit, Hare], [Hare], [Rabbit, Hare]}, + [Rabbit, Hare]), + assert_not_clustered(Bunny). + +update_cluster_nodes(Config) -> + [Rabbit, Hare, Bunny] = cluster_members(Config), + + %% Mnesia is running... + assert_failure(fun () -> update_cluster_nodes(Rabbit, Hare) end), + + ok = stop_app(Rabbit), + ok = join_cluster(Rabbit, Hare), + ok = stop_app(Bunny), + ok = join_cluster(Bunny, Hare), + ok = start_app(Bunny), + stop_reset_start(Hare), + assert_failure(fun () -> start_app(Rabbit) end), + %% Bogus node + assert_failure(fun () -> update_cluster_nodes(Rabbit, non@existent) end), + %% Inconsistent node + assert_failure(fun () -> update_cluster_nodes(Rabbit, Hare) end), + ok = update_cluster_nodes(Rabbit, Bunny), + ok = start_app(Rabbit), + assert_not_clustered(Hare), + assert_clustered([Rabbit, Bunny]). + +classic_config_discovery_node_list(Config) -> + [Rabbit, Hare] = cluster_members(Config), + + ok = stop_app(Hare), + ok = reset(Hare), + ok = rpc:call(Hare, application, set_env, + [rabbit, cluster_nodes, {[Rabbit], disc}]), + ok = start_app(Hare), + assert_clustered([Rabbit, Hare]), + + ok = stop_app(Hare), + ok = reset(Hare), + ok = rpc:call(Hare, application, set_env, + [rabbit, cluster_nodes, {[Rabbit], ram}]), + ok = start_app(Hare), + assert_cluster_status({[Rabbit, Hare], [Rabbit], [Rabbit, Hare]}, + [Rabbit, Hare]), + + %% List of nodes [node()] is equivalent to {[node()], disk} + ok = stop_app(Hare), + ok = reset(Hare), + ok = rpc:call(Hare, application, set_env, + [rabbit, cluster_nodes, [Rabbit]]), + ok = start_app(Hare), + assert_clustered([Rabbit, Hare]), + + ok = stop_app(Hare), + ok = reset(Hare), + %% If we use an invalid cluster_nodes conf, the node fails to start. + ok = rpc:call(Hare, application, set_env, + [rabbit, cluster_nodes, "Yes, please"]), + assert_failure(fun () -> start_app(Hare) end), + assert_not_clustered(Rabbit). + +force_reset_node(Config) -> + [Rabbit, Hare, _Bunny] = cluster_members(Config), + + stop_join_start(Rabbit, Hare), + stop_app(Rabbit), + force_reset(Rabbit), + %% Hare thinks that Rabbit is still clustered + assert_cluster_status({[Rabbit, Hare], [Rabbit, Hare], [Hare]}, + [Hare]), + %% %% ...but it isn't + assert_cluster_status({[Rabbit], [Rabbit], []}, [Rabbit]), + %% We can rejoin Rabbit and Hare + update_cluster_nodes(Rabbit, Hare), + start_app(Rabbit), + assert_clustered([Rabbit, Hare]). + +status_with_alarm(Config) -> + [Rabbit, Hare] = rabbit_ct_broker_helpers:get_node_configs(Config, + nodename), + + %% Given: an alarm is raised each node. + rabbit_ct_broker_helpers:rabbitmqctl(Config, Rabbit, + ["set_vm_memory_high_watermark", "0.000000001"]), + rabbit_ct_broker_helpers:rabbitmqctl(Config, Hare, + ["set_disk_free_limit", "2048G"]), + + %% When: we ask for alarm status + S = rabbit_ct_broker_helpers:rpc(Config, Rabbit, + rabbit_alarm, get_alarms, []), + R = rabbit_ct_broker_helpers:rpc(Config, Hare, + rabbit_alarm, get_alarms, []), + + %% Then: both nodes have printed alarm information for eachother. + ok = alarm_information_on_each_node(S, Rabbit, Hare), + ok = alarm_information_on_each_node(R, Rabbit, Hare). + +alarm_information_on_each_node(Result, Rabbit, Hare) -> + %% Example result: + %% [{{resource_limit,disk,'rmq-ct-status_with_alarm-2-24240@localhost'}, + %% []}, + %% {{resource_limit,memory,'rmq-ct-status_with_alarm-1-24120@localhost'}, + %% []}] + Alarms = [A || {A, _} <- Result], + ?assert(lists:member({resource_limit, memory, Rabbit}, Alarms)), + ?assert(lists:member({resource_limit, disk, Hare}, Alarms)), + + ok. + +pid_file_and_await_node_startup(Config) -> + [Rabbit, Hare] = rabbit_ct_broker_helpers:get_node_configs(Config, + nodename), + RabbitConfig = rabbit_ct_broker_helpers:get_node_config(Config,Rabbit), + RabbitPidFile = ?config(pid_file, RabbitConfig), + %% ensure pid file is readable + {ok, _} = file:read_file(RabbitPidFile), + %% ensure wait works on running node + {ok, _} = rabbit_ct_broker_helpers:rabbitmqctl(Config, Rabbit, + ["wait", RabbitPidFile]), + %% stop both nodes + ok = rabbit_ct_broker_helpers:stop_node(Config, Rabbit), + ok = rabbit_ct_broker_helpers:stop_node(Config, Hare), + %% starting first node fails - it was not the last node to stop + {error, _} = rabbit_ct_broker_helpers:start_node(Config, Rabbit), + PreviousPid = pid_from_file(RabbitPidFile), + %% start first node in the background + spawn_link(fun() -> + rabbit_ct_broker_helpers:start_node(Config, Rabbit) + end), + Attempts = 200, + Timeout = 50, + wait_for_pid_file_to_change(RabbitPidFile, PreviousPid, Attempts, Timeout), + {error, _, _} = rabbit_ct_broker_helpers:rabbitmqctl(Config, Rabbit, + ["wait", RabbitPidFile]). + +await_running_count(Config) -> + [Rabbit, Hare] = rabbit_ct_broker_helpers:get_node_configs(Config, + nodename), + RabbitConfig = rabbit_ct_broker_helpers:get_node_config(Config,Rabbit), + RabbitPidFile = ?config(pid_file, RabbitConfig), + {ok, _} = rabbit_ct_broker_helpers:rabbitmqctl(Config, Rabbit, + ["wait", RabbitPidFile]), + %% stop both nodes + ok = rabbit_ct_broker_helpers:stop_node(Config, Hare), + ok = rabbit_ct_broker_helpers:stop_node(Config, Rabbit), + %% start one node in the background + rabbit_ct_broker_helpers:start_node(Config, Rabbit), + {ok, _} = rabbit_ct_broker_helpers:rabbitmqctl(Config, Rabbit, + ["wait", RabbitPidFile]), + ?assertEqual(ok, rabbit_ct_broker_helpers:rpc(Config, Rabbit, + rabbit_nodes, + await_running_count, [1, 30000])), + ?assertEqual({error, timeout}, + rabbit_ct_broker_helpers:rpc(Config, Rabbit, + rabbit_nodes, + await_running_count, [2, 1000])), + ?assertEqual({error, timeout}, + rabbit_ct_broker_helpers:rpc(Config, Rabbit, + rabbit_nodes, + await_running_count, [5, 1000])), + rabbit_ct_broker_helpers:start_node(Config, Hare), + %% this now succeeds + ?assertEqual(ok, rabbit_ct_broker_helpers:rpc(Config, Rabbit, + rabbit_nodes, + await_running_count, [2, 30000])), + %% this still succeeds + ?assertEqual(ok, rabbit_ct_broker_helpers:rpc(Config, Rabbit, + rabbit_nodes, + await_running_count, [1, 30000])), + %% this still fails + ?assertEqual({error, timeout}, + rabbit_ct_broker_helpers:rpc(Config, Rabbit, + rabbit_nodes, + await_running_count, [5, 1000])). + +%% ---------------------------------------------------------------------------- +%% Internal utils +%% ---------------------------------------------------------------------------- + +wait_for_pid_file_to_change(_, 0, _, _) -> + error(timeout_waiting_for_pid_file_to_have_running_pid); +wait_for_pid_file_to_change(PidFile, PreviousPid, Attempts, Timeout) -> + Pid = pid_from_file(PidFile), + case Pid =/= undefined andalso Pid =/= PreviousPid of + true -> ok; + false -> + ct:sleep(Timeout), + wait_for_pid_file_to_change(PidFile, + PreviousPid, + Attempts - 1, + Timeout) + end. + +pid_from_file(PidFile) -> + case file:read_file(PidFile) of + {ok, Content} -> + string:strip(binary_to_list(Content), both, $\n); + {error, enoent} -> + undefined + end. + +cluster_members(Config) -> + rabbit_ct_broker_helpers:get_node_configs(Config, nodename). + +assert_cluster_status(Status0, Nodes) -> + Status = {AllNodes, _, _} = sort_cluster_status(Status0), + wait_for_cluster_status(Status, AllNodes, Nodes). + +wait_for_cluster_status(Status, AllNodes, Nodes) -> + Max = 10000 / ?LOOP_RECURSION_DELAY, + wait_for_cluster_status(0, Max, Status, AllNodes, Nodes). + +wait_for_cluster_status(N, Max, Status, _AllNodes, Nodes) when N >= Max -> + erlang:error({cluster_status_max_tries_failed, + [{nodes, Nodes}, + {expected_status, Status}, + {max_tried, Max}]}); +wait_for_cluster_status(N, Max, Status, AllNodes, Nodes) -> + case lists:all(fun (Node) -> + verify_status_equal(Node, Status, AllNodes) + end, Nodes) of + true -> ok; + false -> timer:sleep(?LOOP_RECURSION_DELAY), + wait_for_cluster_status(N + 1, Max, Status, AllNodes, Nodes) + end. + +verify_status_equal(Node, Status, AllNodes) -> + NodeStatus = sort_cluster_status(cluster_status(Node)), + (AllNodes =/= [Node]) =:= rpc:call(Node, rabbit_mnesia, is_clustered, []) + andalso NodeStatus =:= Status. + +cluster_status(Node) -> + {rpc:call(Node, rabbit_mnesia, cluster_nodes, [all]), + rpc:call(Node, rabbit_mnesia, cluster_nodes, [disc]), + rpc:call(Node, rabbit_mnesia, cluster_nodes, [running])}. + +sort_cluster_status({All, Disc, Running}) -> + {lists:sort(All), lists:sort(Disc), lists:sort(Running)}. + +assert_clustered(Nodes) -> + assert_cluster_status({Nodes, Nodes, Nodes}, Nodes). + +assert_not_clustered(Node) -> + assert_cluster_status({[Node], [Node], [Node]}, [Node]). + +assert_failure(Fun) -> + case catch Fun() of + {error, _Code, Reason} -> Reason; + {error, Reason} -> Reason; + {error_string, Reason} -> Reason; + {badrpc, {'EXIT', Reason}} -> Reason; + %% Failure to start an app result in node shutdown + {badrpc, nodedown} -> nodedown; + {badrpc_multi, Reason, _Nodes} -> Reason; + Other -> error({expected_failure, Other}) + end. + +stop_app(Node) -> + rabbit_control_helper:command(stop_app, Node). + +start_app(Node) -> + rabbit_control_helper:command(start_app, Node). + +join_cluster(Node, To) -> + join_cluster(Node, To, false). + +join_cluster(Node, To, Ram) -> + rabbit_control_helper:command_with_output(join_cluster, Node, [atom_to_list(To)], [{"--ram", Ram}]). + +reset(Node) -> + rabbit_control_helper:command(reset, Node). + +force_reset(Node) -> + rabbit_control_helper:command(force_reset, Node). + +forget_cluster_node(Node, Removee, RemoveWhenOffline) -> + rabbit_control_helper:command(forget_cluster_node, Node, [atom_to_list(Removee)], + [{"--offline", RemoveWhenOffline}]). + +forget_cluster_node(Node, Removee) -> + forget_cluster_node(Node, Removee, false). + +change_cluster_node_type(Node, Type) -> + rabbit_control_helper:command(change_cluster_node_type, Node, [atom_to_list(Type)]). + +update_cluster_nodes(Node, DiscoveryNode) -> + rabbit_control_helper:command(update_cluster_nodes, Node, [atom_to_list(DiscoveryNode)]). + +stop_join_start(Node, ClusterTo, Ram) -> + ok = stop_app(Node), + ok = join_cluster(Node, ClusterTo, Ram), + ok = start_app(Node). + +stop_join_start(Node, ClusterTo) -> + stop_join_start(Node, ClusterTo, false). + +stop_reset_start(Node) -> + ok = stop_app(Node), + ok = reset(Node), + ok = start_app(Node). + +declare(Ch, Name) -> + Res = amqp_channel:call(Ch, #'queue.declare'{durable = true, + queue = Name}), + amqp_channel:call(Ch, #'queue.bind'{queue = Name, + exchange = <<"amq.fanout">>}), + Res. diff --git a/deps/rabbit/test/config_schema_SUITE.erl b/deps/rabbit/test/config_schema_SUITE.erl new file mode 100644 index 0000000000..c538736897 --- /dev/null +++ b/deps/rabbit/test/config_schema_SUITE.erl @@ -0,0 +1,54 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2016-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(config_schema_SUITE). + +-compile(export_all). + +all() -> + [ + run_snippets + ]. + +%% ------------------------------------------------------------------- +%% Testsuite setup/teardown. +%% ------------------------------------------------------------------- + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + Config1 = rabbit_ct_helpers:run_setup_steps(Config), + rabbit_ct_config_schema:init_schemas(rabbit, Config1). + + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_started(Config, Testcase), + Config1 = rabbit_ct_helpers:set_config(Config, [ + {rmq_nodename_suffix, Testcase} + ]), + rabbit_ct_helpers:run_steps(Config1, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()). + +end_per_testcase(Testcase, Config) -> + Config1 = rabbit_ct_helpers:run_steps(Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()), + rabbit_ct_helpers:testcase_finished(Config1, Testcase). + +%% ------------------------------------------------------------------- +%% Testcases. +%% ------------------------------------------------------------------- + +run_snippets(Config) -> + ok = rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, run_snippets1, [Config]). + +run_snippets1(Config) -> + rabbit_ct_config_schema:run_snippets(Config). diff --git a/deps/rabbit/test/config_schema_SUITE_data/certs/cacert.pem b/deps/rabbit/test/config_schema_SUITE_data/certs/cacert.pem new file mode 100644 index 0000000000..eaf6b67806 --- /dev/null +++ b/deps/rabbit/test/config_schema_SUITE_data/certs/cacert.pem @@ -0,0 +1 @@ +I'm not a certificate diff --git a/deps/rabbit/test/config_schema_SUITE_data/certs/cert.pem b/deps/rabbit/test/config_schema_SUITE_data/certs/cert.pem new file mode 100644 index 0000000000..eaf6b67806 --- /dev/null +++ b/deps/rabbit/test/config_schema_SUITE_data/certs/cert.pem @@ -0,0 +1 @@ +I'm not a certificate diff --git a/deps/rabbit/test/config_schema_SUITE_data/certs/key.pem b/deps/rabbit/test/config_schema_SUITE_data/certs/key.pem new file mode 100644 index 0000000000..eaf6b67806 --- /dev/null +++ b/deps/rabbit/test/config_schema_SUITE_data/certs/key.pem @@ -0,0 +1 @@ +I'm not a certificate diff --git a/deps/rabbit/test/config_schema_SUITE_data/rabbit.snippets b/deps/rabbit/test/config_schema_SUITE_data/rabbit.snippets new file mode 100644 index 0000000000..c6ac600dcc --- /dev/null +++ b/deps/rabbit/test/config_schema_SUITE_data/rabbit.snippets @@ -0,0 +1,797 @@ +[{internal_auth_backend, + "auth_backends.1 = internal", + [{rabbit,[{auth_backends,[rabbit_auth_backend_internal]}]}], + []}, + {ldap_auth_backend, + "auth_backends.1 = ldap", + [{rabbit,[{auth_backends,[rabbit_auth_backend_ldap]}]}], + []}, + {multiple_auth_backends, + "auth_backends.1 = ldap +auth_backends.2 = internal", + [{rabbit, + [{auth_backends, + [rabbit_auth_backend_ldap,rabbit_auth_backend_internal]}]}], + []}, + {full_name_auth_backend, + "auth_backends.1 = ldap +# uses module name instead of a short alias, \"http\" +auth_backends.2 = rabbit_auth_backend_http", + [{rabbit, + [{auth_backends,[rabbit_auth_backend_ldap,rabbit_auth_backend_http]}]}], + []}, + {third_party_auth_backend, + "auth_backends.1.authn = internal +# uses module name because this backend is from a 3rd party +auth_backends.1.authz = rabbit_auth_backend_ip_range", + [{rabbit, + [{auth_backends, + [{rabbit_auth_backend_internal,rabbit_auth_backend_ip_range}]}]}], + []}, + {authn_authz_backend, + "auth_backends.1.authn = ldap +auth_backends.1.authz = internal", + [{rabbit, + [{auth_backends, + [{rabbit_auth_backend_ldap,rabbit_auth_backend_internal}]}]}], + []}, + {authn_authz_multiple_backends, + "auth_backends.1.authn = ldap +auth_backends.1.authz = internal +auth_backends.2 = internal", + [{rabbit, + [{auth_backends, + [{rabbit_auth_backend_ldap,rabbit_auth_backend_internal}, + rabbit_auth_backend_internal]}]}], + []}, + {authn_backend_only, + "auth_backends.1.authn = ldap", + [{rabbit, + [{auth_backends, + [{rabbit_auth_backend_ldap,rabbit_auth_backend_ldap}]}]}], + []}, + {ssl_options, + "ssl_options.cacertfile = test/config_schema_SUITE_data/certs/cacert.pem +ssl_options.certfile = test/config_schema_SUITE_data/certs/cert.pem +ssl_options.keyfile = test/config_schema_SUITE_data/certs/key.pem +ssl_options.verify = verify_peer +ssl_options.fail_if_no_peer_cert = true", + [{rabbit, + [{ssl_options, + [{cacertfile,"test/config_schema_SUITE_data/certs/cacert.pem"}, + {certfile,"test/config_schema_SUITE_data/certs/cert.pem"}, + {keyfile,"test/config_schema_SUITE_data/certs/key.pem"}, + {verify,verify_peer}, + {fail_if_no_peer_cert,true}]}]}], + []}, + {tcp_listener, + "listeners.tcp.default = 5673", + [{rabbit,[{tcp_listeners,[5673]}]}],[]}, + {ssl_listener, + "listeners.ssl = none",[{rabbit,[{ssl_listeners,[]}]}],[]}, + {num_acceptors, + "num_acceptors.ssl = 1",[{rabbit,[{num_ssl_acceptors,1}]}],[]}, + + {socket_writer_gc_threshold, + "socket_writer.gc_threshold = 999666111", [{rabbit, [{writer_gc_threshold, 999666111}]}],[]}, + + {socket_writer_gc_threshold_off, + "socket_writer.gc_threshold = off", [{rabbit, [{writer_gc_threshold, undefined}]}],[]}, + + {default_user_settings, + "default_user = guest +default_pass = guest +default_user_tags.administrator = true +default_permissions.configure = .* +default_permissions.read = .* +default_permissions.write = .*", + [{rabbit, + [{default_user,<<"guest">>}, + {default_pass,<<"guest">>}, + {default_user_tags,[administrator]}, + {default_permissions,[<<".*">>,<<".*">>,<<".*">>]}]}], + []}, + {cluster_formation, + "cluster_formation.peer_discovery_backend = rabbit_peer_discovery_classic_config +cluster_formation.classic_config.nodes.peer1 = rabbit@hostname1 +cluster_formation.classic_config.nodes.peer2 = rabbit@hostname2 +cluster_formation.node_type = disc", + [{rabbit, + [{cluster_formation, + [{peer_discovery_backend,rabbit_peer_discovery_classic_config}, + {node_type,disc}]}, + {cluster_nodes,{[rabbit@hostname2,rabbit@hostname1],disc}}]}], + []}, + + {cluster_formation_module_classic_confog_alias, + "cluster_formation.peer_discovery_backend = classic_config +cluster_formation.classic_config.nodes.peer1 = rabbit@hostname1 +cluster_formation.classic_config.nodes.peer2 = rabbit@hostname2", + [{rabbit, + [{cluster_formation, + [{peer_discovery_backend,rabbit_peer_discovery_classic_config}]}, + {cluster_nodes,{[rabbit@hostname2,rabbit@hostname1],disc}}]}], + []}, + + {cluster_formation_module_dns_alias, + "cluster_formation.peer_discovery_backend = dns +cluster_formation.dns.hostname = discovery.eng.example.local", + [{rabbit, + [ + {cluster_formation, + [{peer_discovery_backend,rabbit_peer_discovery_dns}, + {peer_discovery_dns, [ + {hostname, <<"discovery.eng.example.local">>} + ]}]} + ]}], + []}, + + {cluster_formation_disk, + "cluster_formation.peer_discovery_backend = rabbit_peer_discovery_classic_config + cluster_formation.classic_config.nodes.peer1 = rabbit@hostname1 + cluster_formation.classic_config.nodes.peer2 = rabbit@hostname2 + cluster_formation.node_type = disk", + [{rabbit, + [{cluster_formation, + [{peer_discovery_backend,rabbit_peer_discovery_classic_config}, + {node_type,disc}]}, + {cluster_nodes,{[rabbit@hostname2,rabbit@hostname1],disc}}]}], + []}, + {cluster_formation_ram_ignored, + "cluster_formation.node_type = ram",[],[]}, + {tcp_listen_options, + "tcp_listen_options.backlog = 128 +tcp_listen_options.nodelay = true +tcp_listen_options.exit_on_close = false", + [{rabbit, + [{tcp_listen_options, + [{backlog,128},{nodelay,true},{exit_on_close,false}]}]}], + []}, + {vm_memory_watermark_absolute, + "vm_memory_high_watermark.absolute = 1073741824", + [{rabbit,[{vm_memory_high_watermark,{absolute,1073741824}}]}], + []}, + {vm_memory_watermark_absolute_units, + "vm_memory_high_watermark.absolute = 1024MB", + [{rabbit,[{vm_memory_high_watermark,{absolute,"1024MB"}}]}], + []}, + {vm_memory_watermark_paging_ratio, + "vm_memory_high_watermark_paging_ratio = 0.75 + vm_memory_high_watermark.relative = 0.4", + [{rabbit, + [{vm_memory_high_watermark_paging_ratio,0.75}, + {vm_memory_high_watermark,0.4}]}], + []}, + {memory_monitor_interval, "memory_monitor_interval = 5000", + [{rabbit, + [{memory_monitor_interval, 5000}]}], + []}, + {vm_memory_calculation_strategy, "vm_memory_calculation_strategy = rss", + [{rabbit, + [{vm_memory_calculation_strategy, rss}]}], + []}, + {vm_memory_calculation_strategy, "vm_memory_calculation_strategy = erlang", + [{rabbit, + [{vm_memory_calculation_strategy, erlang}]}], + []}, + {vm_memory_calculation_strategy, "vm_memory_calculation_strategy = allocated", + [{rabbit, + [{vm_memory_calculation_strategy, allocated}]}], + []}, + {vm_memory_calculation_strategy, "vm_memory_calculation_strategy = legacy", + [{rabbit, + [{vm_memory_calculation_strategy, legacy}]}], + []}, + {total_memory_available_override_value, + "total_memory_available_override_value = 1000000000", + [{rabbit,[{total_memory_available_override_value, 1000000000}]}], + []}, + {total_memory_available_override_value_units, + "total_memory_available_override_value = 1024MB", + [{rabbit,[{total_memory_available_override_value, "1024MB"}]}], + []}, + {connection_max, + "connection_max = 999", + [{rabbit,[{connection_max, 999}]}], + []}, + {connection_max, + "connection_max = infinity", + [{rabbit,[{connection_max, infinity}]}], + []}, + {channel_max, + "channel_max = 16", + [{rabbit,[{channel_max, 16}]}], + []}, + {max_message_size, + "max_message_size = 131072", + [{rabbit, [{max_message_size, 131072}]}], + []}, + {listeners_tcp_ip, + "listeners.tcp.1 = 192.168.1.99:5672", + [{rabbit,[{tcp_listeners,[{"192.168.1.99",5672}]}]}], + []}, + {listeners_tcp_ip_multiple, + "listeners.tcp.1 = 127.0.0.1:5672 + listeners.tcp.2 = ::1:5672", + [{rabbit,[{tcp_listeners,[{"127.0.0.1",5672},{"::1",5672}]}]}], + []}, + {listeners_tcp_ip_all,"listeners.tcp.1 = :::5672", + [{rabbit,[{tcp_listeners,[{"::",5672}]}]}], + []}, + {listeners_tcp_ipv6, + "listeners.tcp.1 = fe80::2acf:e9ff:fe17:f97b:5672", + [{rabbit,[{tcp_listeners,[{"fe80::2acf:e9ff:fe17:f97b",5672}]}]}], + []}, + {tcp_options_sndbuf, + "tcp_listen_options.backlog = 128 + tcp_listen_options.nodelay = true + tcp_listen_options.sndbuf = 196608 + tcp_listen_options.recbuf = 196608", + [{rabbit, + [{tcp_listen_options, + [{backlog,128},{nodelay,true},{sndbuf,196608},{recbuf,196608}]}]}], + []}, + {tcp_listen_options_nodelay_with_kernel, + "tcp_listen_options.backlog = 4096 + tcp_listen_options.nodelay = true", + [{kernel, + [{inet_default_connect_options,[{nodelay,true}]}, + {inet_default_listen_options,[{nodelay,true}]}]}], + [{kernel, + [{inet_default_connect_options,[{nodelay,true}]}, + {inet_default_listen_options,[{nodelay,true}]}]}, + {rabbit,[{tcp_listen_options,[{backlog,4096},{nodelay,true}]}]}], + []}, + {tcp_listen_options_nodelay, + "tcp_listen_options.backlog = 4096 + tcp_listen_options.nodelay = true", + [{rabbit,[{tcp_listen_options,[{backlog,4096},{nodelay,true}]}]}], + []}, + {ssl_handshake_timeout, + "ssl_handshake_timeout = 10000", + [{rabbit,[{ssl_handshake_timeout,10000}]}], + []}, + {cluster_partition_handling_pause_if_all_down, + "cluster_partition_handling = pause_if_all_down + + ## Recover strategy. Can be either 'autoheal' or 'ignore' + cluster_partition_handling.pause_if_all_down.recover = ignore + + ## Node names to check + cluster_partition_handling.pause_if_all_down.nodes.1 = rabbit@myhost1 + cluster_partition_handling.pause_if_all_down.nodes.2 = rabbit@myhost2", + [{rabbit, + [{cluster_partition_handling, + {pause_if_all_down,[rabbit@myhost2,rabbit@myhost1],ignore}}]}], + []}, + {cluster_partition_handling_autoheal, + "cluster_partition_handling = autoheal", + [{rabbit,[{cluster_partition_handling,autoheal}]}], + []}, + {password_hashing, + "password_hashing_module = rabbit_password_hashing_sha512", + [{rabbit,[{password_hashing_module,rabbit_password_hashing_sha512}]}], + []}, + {ssl_options_verify_peer, + "listeners.ssl.1 = 5671 + ssl_options.cacertfile = test/config_schema_SUITE_data/certs/cacert.pem + ssl_options.certfile = test/config_schema_SUITE_data/certs/cert.pem + ssl_options.keyfile = test/config_schema_SUITE_data/certs/key.pem + ssl_options.verify = verify_peer + ssl_options.fail_if_no_peer_cert = false", + [{rabbit, + [{ssl_listeners,[5671]}, + {ssl_options, + [{cacertfile,"test/config_schema_SUITE_data/certs/cacert.pem"}, + {certfile,"test/config_schema_SUITE_data/certs/cert.pem"}, + {keyfile,"test/config_schema_SUITE_data/certs/key.pem"}, + {verify,verify_peer}, + {fail_if_no_peer_cert,false}]}]}], + []}, + {ssl_options_password, + "listeners.ssl.1 = 5671 + ssl_options.cacertfile = test/config_schema_SUITE_data/certs/cacert.pem + ssl_options.certfile = test/config_schema_SUITE_data/certs/cert.pem + ssl_options.keyfile = test/config_schema_SUITE_data/certs/key.pem + ssl_options.password = t0p$3kRe7", + [{rabbit, + [{ssl_listeners,[5671]}, + {ssl_options, + [{cacertfile,"test/config_schema_SUITE_data/certs/cacert.pem"}, + {certfile,"test/config_schema_SUITE_data/certs/cert.pem"}, + {keyfile,"test/config_schema_SUITE_data/certs/key.pem"}, + {password,"t0p$3kRe7"}]}]}], + []}, + {ssl_options_tls_ver_old, + "listeners.ssl.1 = 5671 + ssl_options.cacertfile = test/config_schema_SUITE_data/certs/cacert.pem + ssl_options.certfile = test/config_schema_SUITE_data/certs/cert.pem + ssl_options.keyfile = test/config_schema_SUITE_data/certs/key.pem + ssl_options.versions.tls1_2 = tlsv1.2 + ssl_options.versions.tls1_1 = tlsv1.1 + ssl_options.versions.tls1 = tlsv1", + [{ssl,[{versions,['tlsv1.2','tlsv1.1',tlsv1]}]}], + [{ssl,[{versions,['tlsv1.2','tlsv1.1',tlsv1]}]}, + {rabbit, + [{ssl_listeners,[5671]}, + {ssl_options, + [{cacertfile,"test/config_schema_SUITE_data/certs/cacert.pem"}, + {certfile,"test/config_schema_SUITE_data/certs/cert.pem"}, + {keyfile,"test/config_schema_SUITE_data/certs/key.pem"}, + {versions,['tlsv1.2','tlsv1.1',tlsv1]}]}]}], + []}, + {ssl_options_tls_ver_new, + "listeners.ssl.1 = 5671 + ssl_options.cacertfile = test/config_schema_SUITE_data/certs/cacert.pem + ssl_options.certfile = test/config_schema_SUITE_data/certs/cert.pem + ssl_options.keyfile = test/config_schema_SUITE_data/certs/key.pem + ssl_options.versions.tls1_2 = tlsv1.2 + ssl_options.versions.tls1_1 = tlsv1.1", + [{ssl,[{versions,['tlsv1.2','tlsv1.1']}]}], + [{ssl,[{versions,['tlsv1.2','tlsv1.1']}]}, + {rabbit, + [{ssl_listeners,[5671]}, + {ssl_options, + [{cacertfile,"test/config_schema_SUITE_data/certs/cacert.pem"}, + {certfile,"test/config_schema_SUITE_data/certs/cert.pem"}, + {keyfile,"test/config_schema_SUITE_data/certs/key.pem"}, + {versions,['tlsv1.2','tlsv1.1']}]}]}], + []}, + + {ssl_options_ciphers, + "listeners.ssl.1 = 5671 + ssl_options.cacertfile = test/config_schema_SUITE_data/certs/cacert.pem + ssl_options.certfile = test/config_schema_SUITE_data/certs/cert.pem + ssl_options.keyfile = test/config_schema_SUITE_data/certs/key.pem + ssl_options.versions.1 = tlsv1.2 + ssl_options.versions.2 = tlsv1.1 + ssl_options.ciphers.1 = ECDHE-ECDSA-AES256-GCM-SHA384 + ssl_options.ciphers.2 = ECDHE-RSA-AES256-GCM-SHA384 + ssl_options.ciphers.3 = ECDHE-ECDSA-AES256-SHA384 + ssl_options.ciphers.4 = ECDHE-RSA-AES256-SHA384 + ssl_options.ciphers.5 = ECDH-ECDSA-AES256-GCM-SHA384 + ssl_options.ciphers.6 = ECDH-RSA-AES256-GCM-SHA384 + ssl_options.ciphers.7 = ECDH-ECDSA-AES256-SHA384 + ssl_options.ciphers.8 = ECDH-RSA-AES256-SHA384 + ssl_options.ciphers.9 = DHE-RSA-AES256-GCM-SHA384", + [{ssl,[{versions,['tlsv1.2','tlsv1.1']}]}], + [{ssl,[{versions,['tlsv1.2','tlsv1.1']}]}, + {rabbit, + [{ssl_listeners,[5671]}, + {ssl_options, + [{cacertfile,"test/config_schema_SUITE_data/certs/cacert.pem"}, + {ciphers, [ + "ECDHE-ECDSA-AES256-GCM-SHA384", + "ECDHE-RSA-AES256-GCM-SHA384", + "ECDHE-ECDSA-AES256-SHA384", + "ECDHE-RSA-AES256-SHA384", + "ECDH-ECDSA-AES256-GCM-SHA384", + "ECDH-RSA-AES256-GCM-SHA384", + "ECDH-ECDSA-AES256-SHA384", + "ECDH-RSA-AES256-SHA384", + "DHE-RSA-AES256-GCM-SHA384" + ]}, + {certfile,"test/config_schema_SUITE_data/certs/cert.pem"}, + {keyfile,"test/config_schema_SUITE_data/certs/key.pem"}, + {versions,['tlsv1.2','tlsv1.1']}]}]}], + []}, + + {ssl_options_allow_poodle, + "listeners.ssl.1 = 5671 + ssl_allow_poodle_attack = true + ssl_options.cacertfile = test/config_schema_SUITE_data/certs/cacert.pem + ssl_options.certfile = test/config_schema_SUITE_data/certs/cert.pem + ssl_options.keyfile = test/config_schema_SUITE_data/certs/key.pem + ssl_options.verify = verify_peer + ssl_options.fail_if_no_peer_cert = false", + [{rabbit, + [{ssl_listeners,[5671]}, + {ssl_allow_poodle_attack,true}, + {ssl_options, + [{cacertfile,"test/config_schema_SUITE_data/certs/cacert.pem"}, + {certfile,"test/config_schema_SUITE_data/certs/cert.pem"}, + {keyfile,"test/config_schema_SUITE_data/certs/key.pem"}, + {verify,verify_peer}, + {fail_if_no_peer_cert,false}]}]}], + []}, + {ssl_options_depth, + "listeners.ssl.1 = 5671 + ssl_options.cacertfile = test/config_schema_SUITE_data/certs/cacert.pem + ssl_options.certfile = test/config_schema_SUITE_data/certs/cert.pem + ssl_options.keyfile = test/config_schema_SUITE_data/certs/key.pem + ssl_options.depth = 2 + ssl_options.verify = verify_peer + ssl_options.fail_if_no_peer_cert = false", + [{rabbit, + [{ssl_listeners,[5671]}, + {ssl_options, + [{cacertfile,"test/config_schema_SUITE_data/certs/cacert.pem"}, + {certfile,"test/config_schema_SUITE_data/certs/cert.pem"}, + {keyfile,"test/config_schema_SUITE_data/certs/key.pem"}, + {depth,2}, + {verify,verify_peer}, + {fail_if_no_peer_cert,false}]}]}], + []}, + {ssl_options_depth_0, + "listeners.ssl.1 = 5671 + ssl_options.cacertfile = test/config_schema_SUITE_data/certs/cacert.pem + ssl_options.certfile = test/config_schema_SUITE_data/certs/cert.pem + ssl_options.keyfile = test/config_schema_SUITE_data/certs/key.pem + ssl_options.depth = 0 + ssl_options.verify = verify_peer + ssl_options.fail_if_no_peer_cert = false", + [{rabbit, + [{ssl_listeners,[5671]}, + {ssl_options, + [{cacertfile,"test/config_schema_SUITE_data/certs/cacert.pem"}, + {certfile,"test/config_schema_SUITE_data/certs/cert.pem"}, + {keyfile,"test/config_schema_SUITE_data/certs/key.pem"}, + {depth,0}, + {verify,verify_peer}, + {fail_if_no_peer_cert,false}]}]}], + []}, + {ssl_options_depth_255, + "listeners.ssl.1 = 5671 + ssl_options.cacertfile = test/config_schema_SUITE_data/certs/cacert.pem + ssl_options.certfile = test/config_schema_SUITE_data/certs/cert.pem + ssl_options.keyfile = test/config_schema_SUITE_data/certs/key.pem + ssl_options.depth = 255 + ssl_options.verify = verify_peer + ssl_options.fail_if_no_peer_cert = false", + [{rabbit, + [{ssl_listeners,[5671]}, + {ssl_options, + [{cacertfile,"test/config_schema_SUITE_data/certs/cacert.pem"}, + {certfile,"test/config_schema_SUITE_data/certs/cert.pem"}, + {keyfile,"test/config_schema_SUITE_data/certs/key.pem"}, + {depth,255}, + {verify,verify_peer}, + {fail_if_no_peer_cert,false}]}]}], + []}, + {ssl_options_honor_cipher_order, + "listeners.ssl.1 = 5671 + ssl_options.cacertfile = test/config_schema_SUITE_data/certs/cacert.pem + ssl_options.certfile = test/config_schema_SUITE_data/certs/cert.pem + ssl_options.keyfile = test/config_schema_SUITE_data/certs/key.pem + ssl_options.depth = 2 + ssl_options.verify = verify_peer + ssl_options.fail_if_no_peer_cert = false + ssl_options.honor_cipher_order = true", + [{rabbit, + [{ssl_listeners,[5671]}, + {ssl_options, + [{cacertfile,"test/config_schema_SUITE_data/certs/cacert.pem"}, + {certfile,"test/config_schema_SUITE_data/certs/cert.pem"}, + {keyfile,"test/config_schema_SUITE_data/certs/key.pem"}, + {depth,2}, + {verify,verify_peer}, + {fail_if_no_peer_cert, false}, + {honor_cipher_order, true}]}]}], + []}, + {ssl_options_honor_ecc_order, + "listeners.ssl.1 = 5671 + ssl_options.cacertfile = test/config_schema_SUITE_data/certs/cacert.pem + ssl_options.certfile = test/config_schema_SUITE_data/certs/cert.pem + ssl_options.keyfile = test/config_schema_SUITE_data/certs/key.pem + ssl_options.depth = 2 + ssl_options.verify = verify_peer + ssl_options.fail_if_no_peer_cert = false + ssl_options.honor_ecc_order = true", + [{rabbit, + [{ssl_listeners,[5671]}, + {ssl_options, + [{cacertfile,"test/config_schema_SUITE_data/certs/cacert.pem"}, + {certfile,"test/config_schema_SUITE_data/certs/cert.pem"}, + {keyfile,"test/config_schema_SUITE_data/certs/key.pem"}, + {depth,2}, + {verify,verify_peer}, + {fail_if_no_peer_cert, false}, + {honor_ecc_order, true}]}]}], + []}, + + {ssl_cert_login_from_cn, + "ssl_cert_login_from = common_name", + [{rabbit,[{ssl_cert_login_from, common_name}]}], + []}, + + {ssl_cert_login_from_dn, + "ssl_cert_login_from = distinguished_name", + [{rabbit,[{ssl_cert_login_from, distinguished_name}]}], + []}, + + {ssl_cert_login_from_san_dns, + "ssl_cert_login_from = subject_alternative_name + ssl_cert_login_san_type = dns + ssl_cert_login_san_index = 0", + [{rabbit,[ + {ssl_cert_login_from, subject_alternative_name}, + {ssl_cert_login_san_type, dns}, + {ssl_cert_login_san_index, 0} + ]}], + []}, + + {tcp_listen_options_linger_on, + "tcp_listen_options.linger.on = true + tcp_listen_options.linger.timeout = 100", + [{rabbit,[{tcp_listen_options,[{linger,{true,100}}]}]}], + []}, + {tcp_listen_options_linger_off, + "tcp_listen_options.linger.on = false + tcp_listen_options.linger.timeout = 100", + [{rabbit,[{tcp_listen_options,[{linger,{false,100}}]}]}], + []}, + {tcp_listen_options_linger_on_notimeout, + "tcp_listen_options.linger.on = true", + [{rabbit,[{tcp_listen_options,[{linger,{true,0}}]}]}], + []}, + {tcp_listen_options_linger_timeout, + "tcp_listen_options.linger.timeout = 100", + [{rabbit,[{tcp_listen_options,[{linger,{false,100}}]}]}], + []}, + + {cluster_formation_randomized_startup_delay_both_values, + "cluster_formation.randomized_startup_delay_range.min = 10 + cluster_formation.randomized_startup_delay_range.max = 30", + [{rabbit, [{cluster_formation, [ + {randomized_startup_delay_range, {10, 30}} + ]}]}], + []}, + + {cluster_formation_randomized_startup_delay_min_only, + "cluster_formation.randomized_startup_delay_range.min = 10", + [{rabbit, [{cluster_formation, [ + {randomized_startup_delay_range, {10, 60}} + ]}]}], + []}, + + {cluster_formation_randomized_startup_delay_max_only, + "cluster_formation.randomized_startup_delay_range.max = 30", + [{rabbit, [{cluster_formation, [ + {randomized_startup_delay_range, {5, 30}} + ]}]}], + []}, + + {cluster_formation_dns, + "cluster_formation.peer_discovery_backend = rabbit_peer_discovery_dns + cluster_formation.dns.hostname = 192.168.0.2.xip.io + cluster_formation.node_type = disc", + [{rabbit, + [{cluster_formation, + [{peer_discovery_dns,[{hostname,<<"192.168.0.2.xip.io">>}]}, + {peer_discovery_backend,rabbit_peer_discovery_dns}, + {node_type,disc}]}]}], + []}, + {cluster_formation_classic, + "cluster_formation.peer_discovery_backend = rabbit_peer_discovery_classic_config + cluster_formation.node_type = disc", + [{rabbit, + [{cluster_formation, + [{peer_discovery_backend,rabbit_peer_discovery_classic_config}, + {node_type,disc}]}]}], + []}, + {cluster_formation_classic_ram, + "cluster_formation.peer_discovery_backend = rabbit_peer_discovery_classic_config + cluster_formation.node_type = ram", + [{rabbit, + [{cluster_formation, + [{peer_discovery_backend,rabbit_peer_discovery_classic_config}, + {node_type,ram}]}]}], + []}, + {background_gc_enabled, + "background_gc_enabled = true + background_gc_target_interval = 30000", + [{rabbit, + [{background_gc_enabled,true},{background_gc_target_interval,30000}]}], + []}, + {background_gc_disabled, + "background_gc_enabled = false + background_gc_target_interval = 30000", + [{rabbit, + [{background_gc_enabled,false},{background_gc_target_interval,30000}]}], + []}, + {credential_validator_length, + "credential_validator.validation_backend = rabbit_credential_validator_min_password_length +credential_validator.min_length = 10", + [{rabbit, + [{credential_validator, + [{validation_backend, + rabbit_credential_validator_min_password_length}, + {min_length,10}]}]}], + []}, + {credential_validator_regexp, + "credential_validator.validation_backend = rabbit_credential_validator_password_regexp +credential_validator.regexp = ^abc\\d+", + [{rabbit, + [{credential_validator, + [{validation_backend,rabbit_credential_validator_password_regexp}, + {regexp,"^abc\\d+"}]}]}], + []}, + {proxy_protocol_on, + "proxy_protocol = true", + [{rabbit,[{proxy_protocol,true}]}],[]}, + {proxy_protocol_off, + "proxy_protocol = false", + [{rabbit,[{proxy_protocol,false}]}],[]}, + {log_debug_file, + "log.file.level = debug", + [{rabbit,[{log, [{file, [{level, debug}]}]}]}], + []}, + {log_debug_console, + "log.console = true + log.console.level = debug", + [{rabbit,[{log, [{console, [{enabled, true}, {level, debug}]}]}]}], + []}, + {log_debug_exchange, + "log.exchange = true + log.exchange.level = debug", + [{rabbit,[{log, [{exchange, [{enabled, true}, {level, debug}]}]}]}], + []}, + {log_debug_syslog, + "log.syslog = true + log.syslog.level = debug", + [{rabbit,[{log, [{syslog, [{enabled, true}, {level, debug}]}]}]}], + []}, + {log_file_name, + "log.file = file_name", + [{rabbit,[{log, [{file, [{file, "file_name"}]}]}]}], + []}, + {log_file_disabled, + "log.file = false", + [{rabbit,[{log, [{file, [{file, false}]}]}]}], + []}, + {log_category_level, + "log.connection.level = debug + log.channel.level = error", + [{rabbit,[{log, [{categories, [{connection, [{level, debug}]}, + {channel, [{level, error}]}]}]}]}], + []}, + {log_category_file, + "log.connection.file = file_name_connection + log.channel.file = file_name_channel", + [{rabbit,[{log, [{categories, [{connection, [{file, "file_name_connection"}]}, + {channel, [{file, "file_name_channel"}]}]}]}]}], + []}, + + {default_worker_pool_size, + "default_worker_pool_size = 512", + [{rabbit, [ + {default_worker_pool_size, 512} + ]}], + []}, + + {delegate_count, + "delegate_count = 64", + [{rabbit, [ + {delegate_count, 64} + ]}], + []}, + + {kernel_net_ticktime, + "net_ticktime = 20", + [{kernel, [ + {net_ticktime, 20} + ]}], + []}, + + {rabbit_consumer_timeout, + "consumer_timeout = 20000", + [{rabbit, [ + {consumer_timeout, 20000} + ]}], + []}, + + {rabbit_msg_store_shutdown_timeout, + "message_store_shutdown_timeout = 600000", + [{rabbit, [ + {msg_store_shutdown_timeout, 600000} + ]}], + []}, + + {rabbit_mnesia_table_loading_retry_timeout, + "mnesia_table_loading_retry_timeout = 45000", + [{rabbit, [ + {mnesia_table_loading_retry_timeout, 45000} + ]}], + []}, + + {log_syslog_settings, + "log.syslog = true + log.syslog.identity = rabbitmq + log.syslog.facility = user + log.syslog.multiline_mode = true + log.syslog.ip = 10.10.10.10 + log.syslog.port = 123", + [ + {rabbit,[{log, [{syslog, [{enabled, true}]}]}]}, + {syslog, [{app_name, "rabbitmq"}, + {facility, user}, + {multiline_mode, true}, + {dest_host, "10.10.10.10"}, + {dest_port, 123}]} + ], + []}, + {log_syslog_tcp, + "log.syslog = true + log.syslog.transport = tcp + log.syslog.protocol = rfc5424 + log.syslog.host = syslog.my-network.com", + [ + {rabbit,[{log, [{syslog, [{enabled, true}]}]}]}, + {syslog, [{protocol, {rfc5424, tcp}}, + {dest_host, "syslog.my-network.com"}]} + ], + []}, + {log_syslog_udp_default, + "log.syslog = true + log.syslog.protocol = rfc3164", + [ + {rabbit,[{log, [{syslog, [{enabled, true}]}]}]}, + {syslog, [{protocol, {rfc3164, udp}}]} + ], + []}, + {log_syslog_tls, + "log.syslog = true + log.syslog.transport = tls + log.syslog.ssl_options.cacertfile = test/config_schema_SUITE_data/certs/cacert.pem + log.syslog.ssl_options.certfile = test/config_schema_SUITE_data/certs/cert.pem + log.syslog.ssl_options.keyfile = test/config_schema_SUITE_data/certs/key.pem + log.syslog.ssl_options.verify = verify_peer + log.syslog.ssl_options.fail_if_no_peer_cert = false", + [{rabbit, [{log, [{syslog, [{enabled, true}]}]}]}, + {syslog, [{protocol, {rfc5424, tls, + [{verify,verify_peer}, + {fail_if_no_peer_cert,false}, + {cacertfile,"test/config_schema_SUITE_data/certs/cacert.pem"}, + {certfile,"test/config_schema_SUITE_data/certs/cert.pem"}, + {keyfile,"test/config_schema_SUITE_data/certs/key.pem"}]}}]}], + []}, + + %% + %% Definitions + %% + + {definition_files, "load_definitions = test/definition_import_SUITE_data/case1.json", + [{rabbit, + [{load_definitions, "test/definition_import_SUITE_data/case1.json"}]}], + []}, + + %% + %% Raft + %% + + {raft_data_dir, + "raft.data_dir = /data/rabbitmq/raft/log", + [{ra, [ + {data_dir, "/data/rabbitmq/raft/log"} + ]}], + []}, + + {raft_segment_max_entries, + "raft.segment_max_entries = 65536", + [{ra, [ + {segment_max_entries, 65536} + ]}], + []}, + + {raft_wal_max_size_bytes, + "raft.wal_max_size_bytes = 1048576", + [{ra, [ + {wal_max_size_bytes, 1048576} + ]}], + []}, + + {raft_wal_max_batch_size, + "raft.wal_max_batch_size = 4096", + [{ra, [ + {wal_max_batch_size, 4096} + ]}], + []}, + + {raft_snapshot_chunk_size, + "raft.snapshot_chunk_size = 1000000", + [{ra, [ + {snapshot_chunk_size, 1000000} + ]}], + []} + +]. diff --git a/deps/rabbit/test/confirms_rejects_SUITE.erl b/deps/rabbit/test/confirms_rejects_SUITE.erl new file mode 100644 index 0000000000..a51253885c --- /dev/null +++ b/deps/rabbit/test/confirms_rejects_SUITE.erl @@ -0,0 +1,412 @@ +-module(confirms_rejects_SUITE). + + +-include_lib("common_test/include/ct.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). +-compile(export_all). + +all() -> + [ + {group, parallel_tests} + ]. + +groups() -> + OverflowTests = [ + confirms_rejects_conflict, + policy_resets_to_default + ], + [ + {parallel_tests, [parallel], [ + {overflow_reject_publish_dlx, [parallel], OverflowTests}, + {overflow_reject_publish, [parallel], OverflowTests}, + dead_queue_rejects, + mixed_dead_alive_queues_reject + ]} + ]. + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + + +init_per_group(overflow_reject_publish, Config) -> + rabbit_ct_helpers:set_config(Config, [ + {overflow, <<"reject-publish">>} + ]); +init_per_group(overflow_reject_publish_dlx, Config) -> + rabbit_ct_helpers:set_config(Config, [ + {overflow, <<"reject-publish-dlx">>} + ]); +init_per_group(Group, Config) -> + ClusterSize = 2, + Config1 = rabbit_ct_helpers:set_config(Config, [ + {rmq_nodename_suffix, Group}, + {rmq_nodes_count, ClusterSize} + ]), + rabbit_ct_helpers:run_steps(Config1, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()). + +end_per_group(overflow_reject_publish, _Config) -> + ok; +end_per_group(overflow_reject_publish_dlx, _Config) -> + ok; +end_per_group(_Group, Config) -> + rabbit_ct_helpers:run_steps(Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()). + +init_per_testcase(policy_resets_to_default = Testcase, Config) -> + Conn = rabbit_ct_client_helpers:open_unmanaged_connection(Config), + rabbit_ct_helpers:testcase_started( + rabbit_ct_helpers:set_config(Config, [{conn, Conn}]), Testcase); +init_per_testcase(Testcase, Config) + when Testcase == confirms_rejects_conflict; + Testcase == dead_queue_rejects; + Testcase == mixed_dead_alive_queues_reject -> + Conn = rabbit_ct_client_helpers:open_unmanaged_connection(Config), + Conn1 = rabbit_ct_client_helpers:open_unmanaged_connection(Config), + + rabbit_ct_helpers:testcase_started( + rabbit_ct_helpers:set_config(Config, [{conn, Conn}, {conn1, Conn1}]), + Testcase). + +end_per_testcase(policy_resets_to_default = Testcase, Config) -> + {_, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + XOverflow = ?config(overflow, Config), + QueueName = <<"policy_resets_to_default", "_", XOverflow/binary>>, + amqp_channel:call(Ch, #'queue.delete'{queue = QueueName}), + rabbit_ct_client_helpers:close_channels_and_connection(Config, 0), + + Conn = ?config(conn, Config), + + rabbit_ct_client_helpers:close_connection(Conn), + + rabbit_ct_helpers:testcase_finished(Config, Testcase); +end_per_testcase(confirms_rejects_conflict = Testcase, Config) -> + {_, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + XOverflow = ?config(overflow, Config), + QueueName = <<"confirms_rejects_conflict", "_", XOverflow/binary>>, + amqp_channel:call(Ch, #'queue.delete'{queue = QueueName}), + end_per_testcase0(Testcase, Config); +end_per_testcase(dead_queue_rejects = Testcase, Config) -> + {_, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + amqp_channel:call(Ch, #'queue.delete'{queue = <<"dead_queue_rejects">>}), + end_per_testcase0(Testcase, Config); +end_per_testcase(mixed_dead_alive_queues_reject = Testcase, Config) -> + {_, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + amqp_channel:call(Ch, #'queue.delete'{queue = <<"mixed_dead_alive_queues_reject_dead">>}), + amqp_channel:call(Ch, #'queue.delete'{queue = <<"mixed_dead_alive_queues_reject_alive">>}), + amqp_channel:call(Ch, #'exchange.delete'{exchange = <<"mixed_dead_alive_queues_reject">>}), + end_per_testcase0(Testcase, Config). + +end_per_testcase0(Testcase, Config) -> + rabbit_ct_client_helpers:close_channels_and_connection(Config, 0), + + Conn = ?config(conn, Config), + Conn1 = ?config(conn1, Config), + + rabbit_ct_client_helpers:close_connection(Conn), + rabbit_ct_client_helpers:close_connection(Conn1), + + clean_acks_mailbox(), + + rabbit_ct_helpers:testcase_finished(Config, Testcase). + +dead_queue_rejects(Config) -> + Conn = ?config(conn, Config), + {ok, Ch} = amqp_connection:open_channel(Conn), + QueueName = <<"dead_queue_rejects">>, + amqp_channel:call(Ch, #'confirm.select'{}), + amqp_channel:register_confirm_handler(Ch, self()), + + amqp_channel:call(Ch, #'queue.declare'{queue = QueueName, + durable = true}), + + amqp_channel:call(Ch, #'basic.publish'{routing_key = QueueName}, + #amqp_msg{payload = <<"HI">>}), + + receive + {'basic.ack',_,_} -> ok + after 10000 -> + error(timeout_waiting_for_initial_ack) + end, + + BasicPublish = #'basic.publish'{routing_key = QueueName}, + AmqpMsg = #amqp_msg{payload = <<"HI">>}, + kill_queue_expect_nack(Config, Ch, QueueName, BasicPublish, AmqpMsg, 5). + +mixed_dead_alive_queues_reject(Config) -> + Conn = ?config(conn, Config), + {ok, Ch} = amqp_connection:open_channel(Conn), + QueueNameDead = <<"mixed_dead_alive_queues_reject_dead">>, + QueueNameAlive = <<"mixed_dead_alive_queues_reject_alive">>, + ExchangeName = <<"mixed_dead_alive_queues_reject">>, + + amqp_channel:call(Ch, #'confirm.select'{}), + amqp_channel:register_confirm_handler(Ch, self()), + + amqp_channel:call(Ch, #'queue.declare'{queue = QueueNameDead, + durable = true}), + amqp_channel:call(Ch, #'queue.declare'{queue = QueueNameAlive, + durable = true}), + + amqp_channel:call(Ch, #'exchange.declare'{exchange = ExchangeName, + durable = true}), + + amqp_channel:call(Ch, #'queue.bind'{exchange = ExchangeName, + queue = QueueNameAlive, + routing_key = <<"route">>}), + + amqp_channel:call(Ch, #'queue.bind'{exchange = ExchangeName, + queue = QueueNameDead, + routing_key = <<"route">>}), + + amqp_channel:call(Ch, #'basic.publish'{exchange = ExchangeName, + routing_key = <<"route">>}, + #amqp_msg{payload = <<"HI">>}), + + receive + {'basic.ack',_,_} -> ok; + {'basic.nack',_,_,_} -> error(expecting_ack_got_nack) + after 50000 -> + error({timeout_waiting_for_initial_ack, process_info(self(), messages)}) + end, + + BasicPublish = #'basic.publish'{exchange = ExchangeName, routing_key = <<"route">>}, + AmqpMsg = #amqp_msg{payload = <<"HI">>}, + kill_queue_expect_nack(Config, Ch, QueueNameDead, BasicPublish, AmqpMsg, 5). + +kill_queue_expect_nack(_Config, _Ch, _QueueName, _BasicPublish, _AmqpMsg, 0) -> + error(expecting_nack_got_ack); +kill_queue_expect_nack(Config, Ch, QueueName, BasicPublish, AmqpMsg, Tries) -> + kill_the_queue(QueueName, Config), + amqp_channel:cast(Ch, BasicPublish, AmqpMsg), + R = receive + {'basic.nack',_,_,_} -> + ok; + {'basic.ack',_,_} -> + retry + after 10000 -> + error({timeout_waiting_for_nack, process_info(self(), messages)}) + end, + case R of + ok -> + ok; + retry -> + kill_queue_expect_nack(Config, Ch, QueueName, BasicPublish, AmqpMsg, Tries - 1) + end. + +confirms_rejects_conflict(Config) -> + Conn = ?config(conn, Config), + Conn1 = ?config(conn1, Config), + + {ok, Ch} = amqp_connection:open_channel(Conn), + {ok, Ch1} = amqp_connection:open_channel(Conn1), + + false = Conn =:= Conn1, + false = Ch =:= Ch1, + + amqp_channel:call(Ch, #'confirm.select'{}), + amqp_channel:register_confirm_handler(Ch, self()), + + XOverflow = ?config(overflow, Config), + QueueName = <<"confirms_rejects_conflict", "_", XOverflow/binary>>, + amqp_channel:call(Ch, #'queue.declare'{queue = QueueName, + durable = true, + arguments = [{<<"x-max-length">>, long, 12}, + {<<"x-overflow">>, longstr, XOverflow}] + }), + %% Consume 3 messages at once. Do that often. + Consume = fun Consume() -> + receive + stop -> ok + after 1 -> + amqp_channel:cast(Ch1, #'basic.get'{queue = QueueName, no_ack = true}), + amqp_channel:cast(Ch1, #'basic.get'{queue = QueueName, no_ack = true}), + amqp_channel:cast(Ch1, #'basic.get'{queue = QueueName, no_ack = true}), + amqp_channel:cast(Ch1, #'basic.get'{queue = QueueName, no_ack = true}), + Consume() + end + end, + + Produce = fun + Produce(0) -> ok; + Produce(N) -> + amqp_channel:call(Ch, #'basic.publish'{routing_key = QueueName}, + #amqp_msg{payload = <<"HI">>}), + Produce(N - 1) + end, + + %% Initial queue should be full + % Produce(20), + + %% Consumer is a separate process. + Consumer = spawn(Consume), + + %% A long run. Should create race conditions hopefully. + Produce(500000), + + Result = validate_acks_mailbox(), + + Consumer ! stop, + % Result. + case Result of + ok -> ok; + {error, E} -> error(E) + end. + +policy_resets_to_default(Config) -> + Conn = ?config(conn, Config), + + {ok, Ch} = amqp_connection:open_channel(Conn), + + amqp_channel:call(Ch, #'confirm.select'{}), + amqp_channel:register_confirm_handler(Ch, self()), + + XOverflow = ?config(overflow, Config), + QueueName = <<"policy_resets_to_default", "_", XOverflow/binary>>, + amqp_channel:call(Ch, #'queue.declare'{queue = QueueName, + durable = true + }), + MaxLength = 2, + rabbit_ct_broker_helpers:set_policy( + Config, 0, + QueueName, QueueName, <<"queues">>, + [{<<"max-length">>, MaxLength}, {<<"overflow">>, XOverflow}]), + + timer:sleep(1000), + + [amqp_channel:call(Ch, #'basic.publish'{routing_key = QueueName}, + #amqp_msg{payload = <<"HI">>}) + || _ <- lists:seq(1, MaxLength)], + + assert_acks(MaxLength), + + #'queue.declare_ok'{message_count = MaxLength} = + amqp_channel:call(Ch, #'queue.declare'{queue = QueueName, + durable = true}), + + RejectedMessage = <<"HI-rejected">>, + amqp_channel:call(Ch, #'basic.publish'{routing_key = QueueName}, + #amqp_msg{payload = RejectedMessage}), + + assert_nack(), + + rabbit_ct_broker_helpers:set_policy( + Config, 0, + QueueName, QueueName, <<"queues">>, + [{<<"max-length">>, MaxLength}]), + + NotRejectedMessage = <<"HI-not-rejected">>, + amqp_channel:call(Ch, #'basic.publish'{routing_key = QueueName}, + #amqp_msg{payload = NotRejectedMessage}), + + assert_ack(), + + #'queue.declare_ok'{message_count = MaxLength} = + amqp_channel:call(Ch, #'queue.declare'{queue = QueueName, + durable = true}), + + Msgs = consume_all_messages(Ch, QueueName), + case {lists:member(RejectedMessage, Msgs), lists:member(NotRejectedMessage, Msgs)} of + {true, _} -> error({message_should_be_rejected, RejectedMessage}); + {_, false} -> error({message_should_be_enqueued, NotRejectedMessage}); + _ -> ok + end. + +consume_all_messages(Ch, QueueName) -> + consume_all_messages(Ch, QueueName, []). + +consume_all_messages(Ch, QueueName, Msgs) -> + case amqp_channel:call(Ch, #'basic.get'{queue = QueueName, no_ack = true}) of + {#'basic.get_ok'{}, #amqp_msg{payload = Msg}} -> + consume_all_messages(Ch, QueueName, [Msg | Msgs]); + #'basic.get_empty'{} -> Msgs + end. + +assert_ack() -> + receive {'basic.ack', _, _} -> ok + after 10000 -> error(timeout_waiting_for_ack) + end, + clean_acks_mailbox(). + +assert_nack() -> + receive {'basic.nack', _, _, _} -> ok + after 10000 -> error(timeout_waiting_for_nack) + end, + clean_acks_mailbox(). + +assert_acks(N) -> + receive {'basic.ack', N, _} -> ok + after 10000 -> error({timeout_waiting_for_ack, N}) + end, + clean_acks_mailbox(). + +validate_acks_mailbox() -> + Result = validate_acks_mailbox({0, ok}), + clean_acks_mailbox(), + Result. + +validate_acks_mailbox({LatestMultipleN, LatestMultipleAck}) -> + Received = receive + {'basic.ack', N, Multiple} = A -> {N, Multiple, A}; + {'basic.nack', N, Multiple, _} = A -> {N, Multiple, A} + after + 10000 -> none + end, + % ct:pal("Received ~p~n", [Received]), + case Received of + {LatestN, IsMultiple, AckOrNack} -> + case LatestN < LatestMultipleN of + true -> + {error, {received_ack_lower_than_latest_multiple, AckOrNack, smaller_than, LatestMultipleAck}}; + false -> + case IsMultiple of + true -> validate_acks_mailbox({LatestN, AckOrNack}); + false -> validate_acks_mailbox({LatestMultipleN, LatestMultipleAck}) + end + end; + none -> ok + end. + +clean_acks_mailbox() -> + receive + {'basic.ack', _, _} -> clean_acks_mailbox(); + {'basic.nack', _, _, _} -> clean_acks_mailbox() + after + 1000 -> done + end. + +kill_the_queue(QueueName, Config) -> + rabbit_ct_broker_helpers:rpc(Config, 0, ?MODULE, kill_the_queue, [QueueName]). + +kill_the_queue(QueueName) -> + [begin + {ok, Q} = rabbit_amqqueue:lookup({resource, <<"/">>, queue, QueueName}), + Pid = amqqueue:get_pid(Q), + ct:pal("~w killed", [Pid]), + timer:sleep(1), + exit(Pid, kill) + end + || _ <- lists:seq(1, 50)], + timer:sleep(1), + {ok, Q} = rabbit_amqqueue:lookup({resource, <<"/">>, queue, QueueName}), + Pid = amqqueue:get_pid(Q), + case is_process_alive(Pid) of + %% Try to kill it again + true -> kill_the_queue(QueueName); + false -> ok + end. + +flush() -> + receive + Any -> + ct:pal("flush ~p", [Any]), + flush() + after 0 -> + ok + end. diff --git a/deps/rabbit/test/consumer_timeout_SUITE.erl b/deps/rabbit/test/consumer_timeout_SUITE.erl new file mode 100644 index 0000000000..468714328d --- /dev/null +++ b/deps/rabbit/test/consumer_timeout_SUITE.erl @@ -0,0 +1,262 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2011-2020 VMware, Inc. or its affiliates. All rights reserved. +%% +-module(consumer_timeout_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("kernel/include/file.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-compile(export_all). + +-define(TIMEOUT, 30000). + +-import(quorum_queue_utils, [wait_for_messages/2]). + +all() -> + [ + {group, parallel_tests} + ]. + +groups() -> + AllTests = [consumer_timeout, + consumer_timeout_basic_get, + consumer_timeout_no_basic_cancel_capability + ], + [ + {parallel_tests, [], + [ + {classic_queue, [parallel], AllTests}, + {mirrored_queue, [parallel], AllTests}, + {quorum_queue, [parallel], AllTests} + ]} + ]. + +suite() -> + [ + {timetrap, {minutes, 7}} + ]. + +%% ------------------------------------------------------------------- +%% Testsuite setup/teardown. +%% ------------------------------------------------------------------- + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(classic_queue, Config) -> + rabbit_ct_helpers:set_config( + Config, + [{queue_args, [{<<"x-queue-type">>, longstr, <<"classic">>}]}, + {queue_durable, true}]); +init_per_group(quorum_queue, Config) -> + case rabbit_ct_broker_helpers:enable_feature_flag(Config, quorum_queue) of + ok -> + rabbit_ct_helpers:set_config( + Config, + [{queue_args, [{<<"x-queue-type">>, longstr, <<"quorum">>}]}, + {queue_durable, true}]); + Skip -> + Skip + end; +init_per_group(mirrored_queue, Config) -> + rabbit_ct_broker_helpers:set_ha_policy(Config, 0, <<"^max_length.*queue">>, + <<"all">>, [{<<"ha-sync-mode">>, <<"automatic">>}]), + Config1 = rabbit_ct_helpers:set_config( + Config, [{is_mirrored, true}, + {queue_args, [{<<"x-queue-type">>, longstr, <<"classic">>}]}, + {queue_durable, true}]), + rabbit_ct_helpers:run_steps(Config1, []); +init_per_group(Group, Config0) -> + case lists:member({group, Group}, all()) of + true -> + ClusterSize = 3, + Config = rabbit_ct_helpers:merge_app_env( + Config0, {rabbit, [{channel_tick_interval, 1000}, + {quorum_tick_interval, 1000}, + {consumer_timeout, 5000}]}), + Config1 = rabbit_ct_helpers:set_config( + Config, [ {rmq_nodename_suffix, Group}, + {rmq_nodes_count, ClusterSize} + ]), + rabbit_ct_helpers:run_steps(Config1, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()); + false -> + rabbit_ct_helpers:run_steps(Config0, []) + end. + +end_per_group(Group, Config) -> + case lists:member({group, Group}, all()) of + true -> + rabbit_ct_helpers:run_steps(Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()); + false -> + Config + end. + +init_per_testcase(Testcase, Config) -> + Group = proplists:get_value(name, ?config(tc_group_properties, Config)), + Q = rabbit_data_coercion:to_binary(io_lib:format("~p_~p", [Group, Testcase])), + Q2 = rabbit_data_coercion:to_binary(io_lib:format("~p_~p_2", [Group, Testcase])), + Config1 = rabbit_ct_helpers:set_config(Config, [{queue_name, Q}, + {queue_name_2, Q2}]), + rabbit_ct_helpers:testcase_started(Config1, Testcase). + +end_per_testcase(Testcase, Config) -> + {_, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + amqp_channel:call(Ch, #'queue.delete'{queue = ?config(queue_name, Config)}), + amqp_channel:call(Ch, #'queue.delete'{queue = ?config(queue_name_2, Config)}), + rabbit_ct_helpers:testcase_finished(Config, Testcase). + +consumer_timeout(Config) -> + {Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + QName = ?config(queue_name, Config), + declare_queue(Ch, Config, QName), + publish(Ch, QName, [<<"msg1">>]), + wait_for_messages(Config, [[QName, <<"1">>, <<"1">>, <<"0">>]]), + subscribe(Ch, QName, false), + erlang:monitor(process, Conn), + erlang:monitor(process, Ch), + receive + {'DOWN', _, process, Ch, _} -> ok + after 30000 -> + flush(1), + exit(channel_exit_expected) + end, + receive + {'DOWN', _, process, Conn, _} -> + flush(1), + exit(unexpected_connection_exit) + after 2000 -> + ok + end, + rabbit_ct_client_helpers:close_channel(Ch), + ok. + +consumer_timeout_basic_get(Config) -> + {Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + QName = ?config(queue_name, Config), + declare_queue(Ch, Config, QName), + publish(Ch, QName, [<<"msg1">>]), + wait_for_messages(Config, [[QName, <<"1">>, <<"1">>, <<"0">>]]), + [_DelTag] = consume(Ch, QName, [<<"msg1">>]), + erlang:monitor(process, Conn), + erlang:monitor(process, Ch), + receive + {'DOWN', _, process, Ch, _} -> ok + after 30000 -> + flush(1), + exit(channel_exit_expected) + end, + receive + {'DOWN', _, process, Conn, _} -> + flush(1), + exit(unexpected_connection_exit) + after 2000 -> + ok + end, + ok. + + +-define(CLIENT_CAPABILITIES, + [{<<"publisher_confirms">>, bool, true}, + {<<"exchange_exchange_bindings">>, bool, true}, + {<<"basic.nack">>, bool, true}, + {<<"consumer_cancel_notify">>, bool, false}, + {<<"connection.blocked">>, bool, true}, + {<<"authentication_failure_close">>, bool, true}]). + +consumer_timeout_no_basic_cancel_capability(Config) -> + Port = rabbit_ct_broker_helpers:get_node_config(Config, 0, tcp_port_amqp), + Props = [{<<"capabilities">>, table, ?CLIENT_CAPABILITIES}], + AmqpParams = #amqp_params_network{port = Port, + host = "localhost", + virtual_host = <<"/">>, + client_properties = Props + }, + {ok, Conn} = amqp_connection:start(AmqpParams), + {ok, Ch} = amqp_connection:open_channel(Conn), + QName = ?config(queue_name, Config), + declare_queue(Ch, Config, QName), + publish(Ch, QName, [<<"msg1">>]), + wait_for_messages(Config, [[QName, <<"1">>, <<"1">>, <<"0">>]]), + erlang:monitor(process, Conn), + erlang:monitor(process, Ch), + subscribe(Ch, QName, false), + receive + {#'basic.deliver'{delivery_tag = _, + redelivered = false}, _} -> + %% do nothing with the delivery should trigger timeout + ok + after 5000 -> + exit(deliver_timeout) + end, + receive + {'DOWN', _, process, Ch, _} -> ok + after 30000 -> + flush(1), + exit(channel_exit_expected) + end, + receive + {'DOWN', _, process, Conn, _} -> + flush(1), + exit(unexpected_connection_exit) + after 2000 -> + ok + end, + ok. +%%%%%%%%%%%%%%%%%%%%%%%% +%% Test helpers +%%%%%%%%%%%%%%%%%%%%%%%% + +declare_queue(Ch, Config, QName) -> + Args = ?config(queue_args, Config), + Durable = ?config(queue_durable, Config), + #'queue.declare_ok'{} = amqp_channel:call(Ch, #'queue.declare'{queue = QName, + arguments = Args, + durable = Durable}). +publish(Ch, QName, Payloads) -> + [amqp_channel:call(Ch, #'basic.publish'{routing_key = QName}, #amqp_msg{payload = Payload}) + || Payload <- Payloads]. + +consume(Ch, QName, Payloads) -> + consume(Ch, QName, false, Payloads). + +consume(Ch, QName, NoAck, Payloads) -> + [begin + {#'basic.get_ok'{delivery_tag = DTag}, #amqp_msg{payload = Payload}} = + amqp_channel:call(Ch, #'basic.get'{queue = QName, + no_ack = NoAck}), + DTag + end || Payload <- Payloads]. + +subscribe(Ch, Queue, NoAck) -> + subscribe(Ch, Queue, NoAck, <<"ctag">>). + +subscribe(Ch, Queue, NoAck, Ctag) -> + amqp_channel:subscribe(Ch, #'basic.consume'{queue = Queue, + no_ack = NoAck, + consumer_tag = Ctag}, + self()), + receive + #'basic.consume_ok'{consumer_tag = Ctag} -> + ok + end. + +flush(T) -> + receive X -> + ct:pal("flushed ~w", [X]), + flush(T) + after T -> + ok + end. diff --git a/deps/rabbit/test/crashing_queues_SUITE.erl b/deps/rabbit/test/crashing_queues_SUITE.erl new file mode 100644 index 0000000000..cf88fb00f0 --- /dev/null +++ b/deps/rabbit/test/crashing_queues_SUITE.erl @@ -0,0 +1,267 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(crashing_queues_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). + +-compile(export_all). + +all() -> + [ + {group, cluster_size_2} + ]. + +groups() -> + [ + {cluster_size_2, [], [ + crashing_unmirrored, + crashing_mirrored, + give_up_after_repeated_crashes + ]} + ]. + +%% ------------------------------------------------------------------- +%% Testsuite setup/teardown. +%% ------------------------------------------------------------------- + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(cluster_size_2, Config) -> + rabbit_ct_helpers:set_config(Config, [ + {rmq_nodes_count, 2} + ]). + +end_per_group(_, Config) -> + Config. + +init_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_started(Config, Testcase), + ClusterSize = ?config(rmq_nodes_count, Config), + TestNumber = rabbit_ct_helpers:testcase_number(Config, ?MODULE, Testcase), + Config1 = rabbit_ct_helpers:set_config(Config, [ + {rmq_nodename_suffix, Testcase}, + {tcp_ports_base, {skip_n_nodes, TestNumber * ClusterSize}} + ]), + rabbit_ct_helpers:run_steps(Config1, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()). + +end_per_testcase(Testcase, Config) -> + Config1 = rabbit_ct_helpers:run_steps(Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()), + rabbit_ct_helpers:testcase_finished(Config1, Testcase). + +%% ------------------------------------------------------------------- +%% Testcases. +%% ------------------------------------------------------------------- + +crashing_unmirrored(Config) -> + [A, B] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + ChA = rabbit_ct_client_helpers:open_channel(Config, A), + ConnB = rabbit_ct_client_helpers:open_connection(Config, B), + QName = <<"crashing_unmirrored-q">>, + amqp_channel:call(ChA, #'confirm.select'{}), + test_queue_failure(A, ChA, ConnB, 1, 0, + #'queue.declare'{queue = QName, durable = true}), + test_queue_failure(A, ChA, ConnB, 0, 0, + #'queue.declare'{queue = QName, durable = false}), + ok. + +crashing_mirrored(Config) -> + [A, B] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + rabbit_ct_broker_helpers:set_ha_policy(Config, A, <<".*">>, <<"all">>), + ChA = rabbit_ct_client_helpers:open_channel(Config, A), + ConnB = rabbit_ct_client_helpers:open_connection(Config, B), + QName = <<"crashing_mirrored-q">>, + amqp_channel:call(ChA, #'confirm.select'{}), + test_queue_failure(A, ChA, ConnB, 2, 1, + #'queue.declare'{queue = QName, durable = true}), + ok. + +test_queue_failure(Node, Ch, RaceConn, MsgCount, FollowerCount, Decl) -> + #'queue.declare_ok'{queue = QName} = amqp_channel:call(Ch, Decl), + try + publish(Ch, QName, transient), + publish(Ch, QName, durable), + Racer = spawn_declare_racer(RaceConn, Decl), + kill_queue(Node, QName), + assert_message_count(MsgCount, Ch, QName), + assert_follower_count(FollowerCount, Node, QName), + stop_declare_racer(Racer) + after + amqp_channel:call(Ch, #'queue.delete'{queue = QName}) + end. + +give_up_after_repeated_crashes(Config) -> + [A, B] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + ChA = rabbit_ct_client_helpers:open_channel(Config, A), + ChB = rabbit_ct_client_helpers:open_channel(Config, B), + QName = <<"give_up_after_repeated_crashes-q">>, + amqp_channel:call(ChA, #'confirm.select'{}), + amqp_channel:call(ChA, #'queue.declare'{queue = QName, + durable = true}), + await_state(A, QName, running), + publish(ChA, QName, durable), + kill_queue_hard(A, QName), + {'EXIT', _} = (catch amqp_channel:call( + ChA, #'queue.declare'{queue = QName, + durable = true})), + await_state(A, QName, crashed), + amqp_channel:call(ChB, #'queue.delete'{queue = QName}), + amqp_channel:call(ChB, #'queue.declare'{queue = QName, + durable = true}), + await_state(A, QName, running), + + %% Since it's convenient, also test absent queue status here. + rabbit_ct_broker_helpers:stop_node(Config, B), + await_state(A, QName, down), + ok. + + +publish(Ch, QName, DelMode) -> + Publish = #'basic.publish'{exchange = <<>>, routing_key = QName}, + Msg = #amqp_msg{props = #'P_basic'{delivery_mode = del_mode(DelMode)}}, + amqp_channel:cast(Ch, Publish, Msg), + amqp_channel:wait_for_confirms(Ch). + +del_mode(transient) -> 1; +del_mode(durable) -> 2. + +spawn_declare_racer(Conn, Decl) -> + Self = self(), + spawn_link(fun() -> declare_racer_loop(Self, Conn, Decl) end). + +stop_declare_racer(Pid) -> + Pid ! stop, + MRef = erlang:monitor(process, Pid), + receive + {'DOWN', MRef, process, Pid, _} -> ok + end. + +declare_racer_loop(Parent, Conn, Decl) -> + receive + stop -> unlink(Parent) + after 0 -> + %% Catch here because we might happen to catch the queue + %% while it is in the middle of recovering and thus + %% explode with NOT_FOUND because crashed. Doesn't matter, + %% we are only in this loop to try to fool the recovery + %% code anyway. + try + case amqp_connection:open_channel(Conn) of + {ok, Ch} -> amqp_channel:call(Ch, Decl); + closing -> ok + end + catch + exit:_ -> + ok + end, + declare_racer_loop(Parent, Conn, Decl) + end. + +await_state(Node, QName, State) -> + await_state(Node, QName, State, 30000). + +await_state(Node, QName, State, Time) -> + case state(Node, QName) of + State -> + ok; + Other -> + case Time of + 0 -> exit({timeout_awaiting_state, State, Other}); + _ -> timer:sleep(100), + await_state(Node, QName, State, Time - 100) + end + end. + +state(Node, QName) -> + V = <<"/">>, + Res = rabbit_misc:r(V, queue, QName), + Infos = rpc:call(Node, rabbit_amqqueue, info_all, [V, [name, state]]), + case Infos of + [] -> undefined; + [[{name, Res}, {state, State}]] -> State + end. + +kill_queue_hard(Node, QName) -> + case kill_queue(Node, QName) of + crashed -> ok; + _NewPid -> timer:sleep(100), + kill_queue_hard(Node, QName) + end. + +kill_queue(Node, QName) -> + Pid1 = queue_pid(Node, QName), + exit(Pid1, boom), + await_new_pid(Node, QName, Pid1). + +queue_pid(Node, QName) -> + Q = lookup(Node, QName), + QPid = amqqueue:get_pid(Q), + State = amqqueue:get_state(Q), + #resource{virtual_host = VHost} = amqqueue:get_name(Q), + case State of + crashed -> + case rabbit_amqqueue_sup_sup:find_for_vhost(VHost, Node) of + {error, {queue_supervisor_not_found, _}} -> {error, no_sup}; + {ok, SPid} -> + case sup_child(Node, SPid) of + {ok, _} -> QPid; %% restarting + {error, no_child} -> crashed %% given up + end + end; + _ -> QPid + end. + +sup_child(Node, Sup) -> + case rpc:call(Node, supervisor2, which_children, [Sup]) of + [{_, Child, _, _}] -> {ok, Child}; + [] -> {error, no_child}; + {badrpc, {'EXIT', {noproc, _}}} -> {error, no_sup} + end. + +lookup(Node, QName) -> + {ok, Q} = rpc:call(Node, rabbit_amqqueue, lookup, + [rabbit_misc:r(<<"/">>, queue, QName)]), + Q. + +await_new_pid(Node, QName, OldPid) -> + case queue_pid(Node, QName) of + OldPid -> timer:sleep(10), + await_new_pid(Node, QName, OldPid); + New -> New + end. + +assert_message_count(Count, Ch, QName) -> + #'queue.declare_ok'{message_count = Count} = + amqp_channel:call(Ch, #'queue.declare'{queue = QName, + passive = true}). + +assert_follower_count(Count, Node, QName) -> + Q = lookup(Node, QName), + [{_, Pids}] = rpc:call(Node, rabbit_amqqueue, info, [Q, [slave_pids]]), + RealCount = case Pids of + '' -> 0; + _ -> length(Pids) + end, + case RealCount of + Count -> + ok; + _ when RealCount < Count -> + timer:sleep(10), + assert_follower_count(Count, Node, QName); + _ -> + exit({too_many_replicas, Count, RealCount}) + end. diff --git a/deps/rabbit/test/dead_lettering_SUITE.erl b/deps/rabbit/test/dead_lettering_SUITE.erl new file mode 100644 index 0000000000..4ee917aa21 --- /dev/null +++ b/deps/rabbit/test/dead_lettering_SUITE.erl @@ -0,0 +1,1174 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2011-2020 VMware, Inc. or its affiliates. All rights reserved. +%% +%% For the full spec see: https://www.rabbitmq.com/dlx.html +%% +-module(dead_lettering_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("kernel/include/file.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-compile(export_all). + +-import(quorum_queue_utils, [wait_for_messages/2]). + +all() -> + [ + {group, dead_letter_tests} + ]. + +groups() -> + DeadLetterTests = [dead_letter_nack, + dead_letter_multiple_nack, + dead_letter_nack_requeue, + dead_letter_nack_requeue_multiple, + dead_letter_reject, + dead_letter_reject_requeue, + dead_letter_max_length_drop_head, + dead_letter_missing_exchange, + dead_letter_routing_key, + dead_letter_routing_key_header_CC, + dead_letter_routing_key_header_BCC, + dead_letter_routing_key_cycle_max_length, + dead_letter_routing_key_cycle_with_reject, + dead_letter_policy, + dead_letter_override_policy, + dead_letter_ignore_policy, + dead_letter_headers, + dead_letter_headers_reason_maxlen, + dead_letter_headers_cycle, + dead_letter_headers_BCC, + dead_letter_headers_CC, + dead_letter_headers_CC_with_routing_key, + dead_letter_headers_first_death], + Opts = [], + [ + {dead_letter_tests, [], + [ + {classic_queue, Opts, DeadLetterTests ++ [dead_letter_ttl, + dead_letter_max_length_reject_publish_dlx, + dead_letter_routing_key_cycle_ttl, + dead_letter_headers_reason_expired, + dead_letter_headers_reason_expired_per_message]}, + {mirrored_queue, Opts, DeadLetterTests ++ [dead_letter_ttl, + dead_letter_max_length_reject_publish_dlx, + dead_letter_routing_key_cycle_ttl, + dead_letter_headers_reason_expired, + dead_letter_headers_reason_expired_per_message]}, + {quorum_queue, Opts, DeadLetterTests} + ]} + ]. + +suite() -> + [ + {timetrap, {minutes, 8}} + ]. + +%% ------------------------------------------------------------------- +%% Testsuite setup/teardown. +%% ------------------------------------------------------------------- + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(classic_queue, Config) -> + rabbit_ct_helpers:set_config( + Config, + [{queue_args, [{<<"x-queue-type">>, longstr, <<"classic">>}]}, + {queue_durable, false}]); +init_per_group(quorum_queue, Config) -> + case rabbit_ct_broker_helpers:enable_feature_flag(Config, quorum_queue) of + ok -> + rabbit_ct_helpers:set_config( + Config, + [{queue_args, [{<<"x-queue-type">>, longstr, <<"quorum">>}]}, + {queue_durable, true}]); + Skip -> + Skip + end; +init_per_group(mirrored_queue, Config) -> + rabbit_ct_broker_helpers:set_ha_policy(Config, 0, <<"^max_length.*queue">>, + <<"all">>, [{<<"ha-sync-mode">>, <<"automatic">>}]), + Config1 = rabbit_ct_helpers:set_config( + Config, [{is_mirrored, true}, + {queue_args, [{<<"x-queue-type">>, longstr, <<"classic">>}]}, + {queue_durable, false}]), + rabbit_ct_helpers:run_steps(Config1, []); +init_per_group(Group, Config) -> + case lists:member({group, Group}, all()) of + true -> + ClusterSize = 3, + Config1 = rabbit_ct_helpers:set_config(Config, [ + {rmq_nodename_suffix, Group}, + {rmq_nodes_count, ClusterSize} + ]), + rabbit_ct_helpers:run_steps(Config1, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()); + false -> + rabbit_ct_helpers:run_steps(Config, []) + end. + +end_per_group(Group, Config) -> + case lists:member({group, Group}, all()) of + true -> + rabbit_ct_helpers:run_steps(Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()); + false -> + Config + end. + +init_per_testcase(Testcase, Config) -> + Group = proplists:get_value(name, ?config(tc_group_properties, Config)), + Q = rabbit_data_coercion:to_binary(io_lib:format("~p_~p", [Group, Testcase])), + Q2 = rabbit_data_coercion:to_binary(io_lib:format("~p_~p_2", [Group, Testcase])), + Policy = rabbit_data_coercion:to_binary(io_lib:format("~p_~p_policy", [Group, Testcase])), + DLXExchange = rabbit_data_coercion:to_binary(io_lib:format("~p_~p_dlx_exchange", + [Group, Testcase])), + Config1 = rabbit_ct_helpers:set_config(Config, [{dlx_exchange, DLXExchange}, + {queue_name, Q}, + {queue_name_dlx, Q2}, + {policy, Policy}]), + rabbit_ct_helpers:testcase_started(Config1, Testcase). + +end_per_testcase(Testcase, Config) -> + {_, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + amqp_channel:call(Ch, #'queue.delete'{queue = ?config(queue_name, Config)}), + amqp_channel:call(Ch, #'queue.delete'{queue = ?config(queue_name_dlx, Config)}), + amqp_channel:call(Ch, #'exchange.delete'{exchange = ?config(dlx_exchange, Config)}), + _ = rabbit_ct_broker_helpers:clear_policy(Config, 0, ?config(policy, Config)), + rabbit_ct_helpers:testcase_finished(Config, Testcase). + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% Dead letter exchanges +%% +%% Messages are dead-lettered when: +%% 1) message is rejected with basic.reject or basic.nack with requeue=false +%% 2) message ttl expires (not implemented in quorum queues) +%% 3) queue length limit is exceeded (only drop-head implemented in quorum queues) +%% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +%% 1) message is rejected with basic.nack, requeue=false and multiple=false +dead_letter_nack(Config) -> + {_Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + QName = ?config(queue_name, Config), + DLXQName = ?config(queue_name_dlx, Config), + declare_dead_letter_queues(Ch, Config, QName, DLXQName), + + P1 = <<"msg1">>, + P2 = <<"msg2">>, + P3 = <<"msg3">>, + + %% Publish 3 messages + publish(Ch, QName, [P1, P2, P3]), + wait_for_messages(Config, [[QName, <<"3">>, <<"3">>, <<"0">>]]), + %% Consume them + [DTag1, DTag2, DTag3] = consume(Ch, QName, [P1, P2, P3]), + %% Nack the last one with multiple = false + amqp_channel:cast(Ch, #'basic.nack'{delivery_tag = DTag3, + multiple = false, + requeue = false}), + wait_for_messages(Config, [[DLXQName, <<"1">>, <<"1">>, <<"0">>]]), + %% Queue is empty + consume_empty(Ch, QName), + %% Consume the last message from the dead letter queue + consume(Ch, DLXQName, [P3]), + consume_empty(Ch, DLXQName), + %% Nack the other two + amqp_channel:cast(Ch, #'basic.nack'{delivery_tag = DTag1, + multiple = false, + requeue = false}), + amqp_channel:cast(Ch, #'basic.nack'{delivery_tag = DTag2, + multiple = false, + requeue = false}), + %% Queue is empty + consume_empty(Ch, QName), + %% Consume the first two messages from the dead letter queue + consume(Ch, DLXQName, [P1, P2]), + consume_empty(Ch, DLXQName). + +%% 1) message is rejected with basic.nack, requeue=false and multiple=true +dead_letter_multiple_nack(Config) -> + {_Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + QName = ?config(queue_name, Config), + DLXQName = ?config(queue_name_dlx, Config), + declare_dead_letter_queues(Ch, Config, QName, DLXQName), + + P1 = <<"msg1">>, + P2 = <<"msg2">>, + P3 = <<"msg3">>, + + %% Publish 3 messages + publish(Ch, QName, [P1, P2, P3]), + wait_for_messages(Config, [[QName, <<"3">>, <<"3">>, <<"0">>]]), + %% Consume them + [_, _, DTag3] = consume(Ch, QName, [P1, P2, P3]), + %% Nack the last one with multiple = true + amqp_channel:cast(Ch, #'basic.nack'{delivery_tag = DTag3, + multiple = true, + requeue = false}), + wait_for_messages(Config, [[DLXQName, <<"3">>, <<"3">>, <<"0">>]]), + %% Consume the 3 messages from the dead letter queue + consume(Ch, DLXQName, [P1, P2, P3]), + consume_empty(Ch, DLXQName), + %% Queue is empty + consume_empty(Ch, QName). + +%% 1) message is rejected with basic.nack, requeue=true and multiple=false. Dead-lettering does not take place +dead_letter_nack_requeue(Config) -> + {_Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + QName = ?config(queue_name, Config), + DLXQName = ?config(queue_name_dlx, Config), + declare_dead_letter_queues(Ch, Config, QName, DLXQName), + + P1 = <<"msg1">>, + P2 = <<"msg2">>, + P3 = <<"msg3">>, + + %% Publish 3 messages + publish(Ch, QName, [P1, P2, P3]), + %% Consume them + wait_for_messages(Config, [[QName, <<"3">>, <<"3">>, <<"0">>]]), + [_, _, DTag3] = consume(Ch, QName, [P1, P2, P3]), + %% Queue is empty + consume_empty(Ch, QName), + %% Nack the last one with multiple = false + amqp_channel:cast(Ch, #'basic.nack'{delivery_tag = DTag3, + multiple = false, + requeue = true}), + %% Consume the last message from the queue + wait_for_messages(Config, [[QName, <<"3">>, <<"1">>, <<"2">>]]), + consume(Ch, QName, [P3]), + consume_empty(Ch, QName), + %% Dead letter queue is empty + consume_empty(Ch, DLXQName). + +%% 1) message is rejected with basic.nack, requeue=true and multiple=true. Dead-lettering does not take place +dead_letter_nack_requeue_multiple(Config) -> + {_Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + QName = ?config(queue_name, Config), + DLXQName = ?config(queue_name_dlx, Config), + declare_dead_letter_queues(Ch, Config, QName, DLXQName), + + P1 = <<"msg1">>, + P2 = <<"msg2">>, + P3 = <<"msg3">>, + + %% Publish 3 messages + publish(Ch, QName, [P1, P2, P3]), + %% Consume them + wait_for_messages(Config, [[QName, <<"3">>, <<"3">>, <<"0">>]]), + [_, _, DTag3] = consume(Ch, QName, [P1, P2, P3]), + %% Queue is empty + consume_empty(Ch, QName), + %% Nack the last one with multiple = true + amqp_channel:cast(Ch, #'basic.nack'{delivery_tag = DTag3, + multiple = true, + requeue = true}), + %% Consume the three messages from the queue + wait_for_messages(Config, [[QName, <<"3">>, <<"3">>, <<"0">>]]), + consume(Ch, QName, [P1, P2, P3]), + consume_empty(Ch, QName), + %% Dead letter queue is empty + consume_empty(Ch, DLXQName). + +%% 1) message is rejected with basic.reject, requeue=false +dead_letter_reject(Config) -> + {_Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + QName = ?config(queue_name, Config), + DLXQName = ?config(queue_name_dlx, Config), + declare_dead_letter_queues(Ch, Config, QName, DLXQName), + + P1 = <<"msg1">>, + P2 = <<"msg2">>, + P3 = <<"msg3">>, + + %% Publish 3 messages + publish(Ch, QName, [P1, P2, P3]), + %% Consume the first message + wait_for_messages(Config, [[QName, <<"3">>, <<"3">>, <<"0">>]]), + [DTag] = consume(Ch, QName, [P1]), + %% Reject it + amqp_channel:cast(Ch, #'basic.reject'{delivery_tag = DTag, + requeue = false}), + %% Consume it from the dead letter queue + wait_for_messages(Config, [[DLXQName, <<"1">>, <<"1">>, <<"0">>]]), + _ = consume(Ch, DLXQName, [P1]), + consume_empty(Ch, DLXQName), + %% Consume the last two from the queue + _ = consume(Ch, QName, [P2, P3]), + consume_empty(Ch, QName). + +%% 1) Message is rejected with basic.reject, requeue=true. Dead-lettering does not take place. +dead_letter_reject_requeue(Config) -> + {_Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + QName = ?config(queue_name, Config), + DLXQName = ?config(queue_name_dlx, Config), + declare_dead_letter_queues(Ch, Config, QName, DLXQName), + + P1 = <<"msg1">>, + P2 = <<"msg2">>, + P3 = <<"msg3">>, + + %% Publish 3 messages + publish(Ch, QName, [P1, P2, P3]), + %% Consume the first one + wait_for_messages(Config, [[QName, <<"3">>, <<"3">>, <<"0">>]]), + [DTag] = consume(Ch, QName, [P1]), + %% Reject the first one + amqp_channel:cast(Ch, #'basic.reject'{delivery_tag = DTag, + requeue = true}), + %% Consume the three messages from the queue + wait_for_messages(Config, [[QName, <<"3">>, <<"3">>, <<"0">>]]), + _ = consume(Ch, QName, [P1, P2, P3]), + consume_empty(Ch, QName), + %% Dead letter is empty + consume_empty(Ch, DLXQName). + +%% 2) Message ttl expires +dead_letter_ttl(Config) -> + {_Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + QName = ?config(queue_name, Config), + DLXQName = ?config(queue_name_dlx, Config), + declare_dead_letter_queues(Ch, Config, QName, DLXQName, [{<<"x-message-ttl">>, long, 1}]), + + %% Publish message + P1 = <<"msg1">>, + publish(Ch, QName, [P1]), + wait_for_messages(Config, [[DLXQName, <<"1">>, <<"1">>, <<"0">>]]), + consume_empty(Ch, QName), + [_] = consume(Ch, DLXQName, [P1]). + +%% 3) The queue length limit is exceeded, message dropped is dead lettered. +%% Default strategy: drop-head +dead_letter_max_length_drop_head(Config) -> + {_Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + QName = ?config(queue_name, Config), + DLXQName = ?config(queue_name_dlx, Config), + + declare_dead_letter_queues(Ch, Config, QName, DLXQName, [{<<"x-max-length">>, long, 1}]), + + P1 = <<"msg1">>, + P2 = <<"msg2">>, + P3 = <<"msg3">>, + + %% Publish 3 messages + publish(Ch, QName, [P1, P2, P3]), + %% Consume the last one from the queue (max-length = 1) + wait_for_messages(Config, [[QName, <<"1">>, <<"1">>, <<"0">>]]), + _ = consume(Ch, QName, [P3]), + consume_empty(Ch, QName), + %% Consume the dropped ones from the dead letter queue + wait_for_messages(Config, [[DLXQName, <<"2">>, <<"2">>, <<"0">>]]), + _ = consume(Ch, DLXQName, [P1, P2]), + consume_empty(Ch, DLXQName). + +%% Another strategy: reject-publish-dlx +dead_letter_max_length_reject_publish_dlx(Config) -> + {_Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + QName = ?config(queue_name, Config), + DLXQName = ?config(queue_name_dlx, Config), + + declare_dead_letter_queues(Ch, Config, QName, DLXQName, + [{<<"x-max-length">>, long, 1}, + {<<"x-overflow">>, longstr, <<"reject-publish-dlx">>}]), + + P1 = <<"msg1">>, + P2 = <<"msg2">>, + P3 = <<"msg3">>, + + %% Publish 3 messages + publish(Ch, QName, [P1, P2, P3]), + %% Consume the first one from the queue (max-length = 1) + wait_for_messages(Config, [[QName, <<"1">>, <<"1">>, <<"0">>]]), + _ = consume(Ch, QName, [P1]), + consume_empty(Ch, QName), + %% Consume the dropped ones from the dead letter queue + wait_for_messages(Config, [[DLXQName, <<"2">>, <<"2">>, <<"0">>]]), + _ = consume(Ch, DLXQName, [P2, P3]), + consume_empty(Ch, DLXQName). + +%% Dead letter exchange does not have to be declared when the queue is declared, but it should +%% exist by the time messages need to be dead-lettered; if it is missing then, the messages will +%% be silently dropped. +dead_letter_missing_exchange(Config) -> + {_Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + Args = ?config(queue_args, Config), + Durable = ?config(queue_durable, Config), + QName = ?config(queue_name, Config), + DLXQName = ?config(queue_name_dlx, Config), + DLXExchange = <<"dlx-exchange-2">>, + #'exchange.delete_ok'{} = amqp_channel:call(Ch, #'exchange.delete'{exchange = DLXExchange}), + + DeadLetterArgs = [{<<"x-max-length">>, long, 1}, + {<<"x-dead-letter-exchange">>, longstr, DLXExchange}, + {<<"x-dead-letter-routing-key">>, longstr, DLXQName}], + #'queue.declare_ok'{} = amqp_channel:call(Ch, #'queue.declare'{queue = QName, arguments = DeadLetterArgs ++ Args, durable = Durable}), + #'queue.declare_ok'{} = amqp_channel:call(Ch, #'queue.declare'{queue = DLXQName, durable = Durable}), + + P1 = <<"msg1">>, + P2 = <<"msg2">>, + + %% Publish one message + publish(Ch, QName, [P1]), + %% Consume it + wait_for_messages(Config, [[QName, <<"1">>, <<"1">>, <<"0">>]]), + [DTag] = consume(Ch, QName, [P1]), + %% Reject it + amqp_channel:cast(Ch, #'basic.reject'{delivery_tag = DTag, + requeue = false}), + wait_for_messages(Config, [[QName, <<"0">>, <<"0">>, <<"0">>]]), + %% Message is not in the dead letter queue (exchange does not exist) + consume_empty(Ch, DLXQName), + + %% Declare the dead-letter exchange + #'exchange.declare_ok'{} = amqp_channel:call(Ch, #'exchange.declare'{exchange = DLXExchange}), + #'queue.bind_ok'{} = amqp_channel:call(Ch, #'queue.bind'{queue = DLXQName, + exchange = DLXExchange, + routing_key = DLXQName}), + + %% Publish another message + publish(Ch, QName, [P2]), + %% Consume it + wait_for_messages(Config, [[QName, <<"1">>, <<"1">>, <<"0">>]]), + [DTag2] = consume(Ch, QName, [P2]), + %% Reject it + amqp_channel:cast(Ch, #'basic.reject'{delivery_tag = DTag2, + requeue = false}), + %% Consume the rejected message from the dead letter queue + wait_for_messages(Config, [[DLXQName, <<"1">>, <<"1">>, <<"0">>]]), + {#'basic.get_ok'{}, #amqp_msg{payload = P2}} = + amqp_channel:call(Ch, #'basic.get'{queue = DLXQName}), + consume_empty(Ch, DLXQName). + +%% +%% ROUTING +%% +%% Dead-lettered messages are routed to their dead letter exchange either: +%% with the routing key specified for the queue they were on; or, +%% if this was not set, (3) with the same routing keys they were originally published with. +%% (4) This includes routing keys added by the CC and BCC headers. +%% +%% 3) All previous tests used a specific key, test the original ones now. +dead_letter_routing_key(Config) -> + {_Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + QName = ?config(queue_name, Config), + DLXQName = ?config(queue_name_dlx, Config), + Args = ?config(queue_args, Config), + Durable = ?config(queue_durable, Config), + DLXExchange = ?config(dlx_exchange, Config), + + %% Do not use a specific key + DeadLetterArgs = [{<<"x-dead-letter-exchange">>, longstr, DLXExchange}], + #'exchange.declare_ok'{} = amqp_channel:call(Ch, #'exchange.declare'{exchange = DLXExchange}), + #'queue.declare_ok'{} = amqp_channel:call(Ch, #'queue.declare'{queue = QName, arguments = DeadLetterArgs ++ Args, durable = Durable}), + #'queue.declare_ok'{} = amqp_channel:call(Ch, #'queue.declare'{queue = DLXQName, durable = Durable}), + + P1 = <<"msg1">>, + P2 = <<"msg2">>, + + %% Publish, consume and nack the first message + publish(Ch, QName, [P1]), + wait_for_messages(Config, [[QName, <<"1">>, <<"1">>, <<"0">>]]), + [DTag1] = consume(Ch, QName, [P1]), + amqp_channel:cast(Ch, #'basic.nack'{delivery_tag = DTag1, + multiple = false, + requeue = false}), + %% Both queues are empty as the message could not been routed in the dlx exchange + wait_for_messages(Config, [[QName, <<"0">>, <<"0">>, <<"0">>]]), + consume_empty(Ch, QName), + consume_empty(Ch, DLXQName), + %% Bind the dlx queue with the original queue routing key + #'queue.bind_ok'{} = amqp_channel:call(Ch, #'queue.bind'{queue = DLXQName, + exchange = DLXExchange, + routing_key = QName}), + %% Publish, consume and nack the second message + publish(Ch, QName, [P2]), + wait_for_messages(Config, [[QName, <<"1">>, <<"1">>, <<"0">>]]), + [DTag2] = consume(Ch, QName, [P2]), + amqp_channel:cast(Ch, #'basic.nack'{delivery_tag = DTag2, + multiple = false, + requeue = false}), + %% Message can now be routed using the recently binded key + wait_for_messages(Config, [[DLXQName, <<"1">>, <<"1">>, <<"0">>]]), + consume(Ch, DLXQName, [P2]), + consume_empty(Ch, QName). + + +%% 4a) If a specific routing key was not set for the queue, use routing keys added by the +%% CC and BCC headers +dead_letter_routing_key_header_CC(Config) -> + {_Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + QName = ?config(queue_name, Config), + DLXQName = ?config(queue_name_dlx, Config), + Args = ?config(queue_args, Config), + Durable = ?config(queue_durable, Config), + DLXExchange = ?config(dlx_exchange, Config), + + %% Do not use a specific key + DeadLetterArgs = [{<<"x-dead-letter-exchange">>, longstr, DLXExchange}], + #'exchange.declare_ok'{} = amqp_channel:call(Ch, #'exchange.declare'{exchange = DLXExchange}), + #'queue.declare_ok'{} = amqp_channel:call(Ch, #'queue.declare'{queue = QName, arguments = DeadLetterArgs ++ Args, durable = Durable}), + #'queue.declare_ok'{} = amqp_channel:call(Ch, #'queue.declare'{queue = DLXQName, durable = Durable}), + #'queue.bind_ok'{} = amqp_channel:call(Ch, #'queue.bind'{queue = DLXQName, + exchange = DLXExchange, + routing_key = DLXQName}), + + P1 = <<"msg1">>, + P2 = <<"msg2">>, + CCHeader = {<<"CC">>, array, [{longstr, DLXQName}]}, + + %% Publish, consume and nack two messages, one with CC header + publish(Ch, QName, [P1]), + publish(Ch, QName, [P2], [CCHeader]), + wait_for_messages(Config, [[QName, <<"2">>, <<"2">>, <<"0">>]]), + [_, DTag2] = consume(Ch, QName, [P1, P2]), + %% P2 is also published to the DLX queue because of the binding to the default exchange + [_] = consume(Ch, DLXQName, [P2]), + amqp_channel:cast(Ch, #'basic.nack'{delivery_tag = DTag2, + multiple = true, + requeue = false}), + %% The second message should have been routed using the CC header + wait_for_messages(Config, [[DLXQName, <<"2">>, <<"1">>, <<"1">>]]), + consume_empty(Ch, QName), + consume(Ch, DLXQName, [P2]), + consume_empty(Ch, DLXQName). + +%% 4b) If a specific routing key was not set for the queue, use routing keys added by the +%% CC and BCC headers +dead_letter_routing_key_header_BCC(Config) -> + {_Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + QName = ?config(queue_name, Config), + DLXQName = ?config(queue_name_dlx, Config), + Args = ?config(queue_args, Config), + Durable = ?config(queue_durable, Config), + DLXExchange = ?config(dlx_exchange, Config), + + %% Do not use a specific key + DeadLetterArgs = [{<<"x-dead-letter-exchange">>, longstr, DLXExchange}], + #'exchange.declare_ok'{} = amqp_channel:call(Ch, #'exchange.declare'{exchange = DLXExchange}), + #'queue.declare_ok'{} = amqp_channel:call(Ch, #'queue.declare'{queue = QName, arguments = DeadLetterArgs ++ Args, durable = Durable}), + #'queue.declare_ok'{} = amqp_channel:call(Ch, #'queue.declare'{queue = DLXQName, durable = Durable}), + #'queue.bind_ok'{} = amqp_channel:call(Ch, #'queue.bind'{queue = DLXQName, + exchange = DLXExchange, + routing_key = DLXQName}), + + P1 = <<"msg1">>, + P2 = <<"msg2">>, + BCCHeader = {<<"BCC">>, array, [{longstr, DLXQName}]}, + + %% Publish, consume and nack two messages, one with BCC header + publish(Ch, QName, [P1]), + publish(Ch, QName, [P2], [BCCHeader]), + wait_for_messages(Config, [[QName, <<"2">>, <<"2">>, <<"0">>]]), + [_, DTag2] = consume(Ch, QName, [P1, P2]), + %% P2 is also published to the DLX queue because of the binding to the default exchange + [_] = consume(Ch, DLXQName, [P2]), + amqp_channel:cast(Ch, #'basic.nack'{delivery_tag = DTag2, + multiple = true, + requeue = false}), + %% The second message should have been routed using the BCC header + wait_for_messages(Config, [[DLXQName, <<"2">>, <<"1">>, <<"1">>]]), + consume_empty(Ch, QName), + consume(Ch, DLXQName, [P2]), + consume_empty(Ch, DLXQName). + +%% It is possible to form a cycle of message dead-lettering. For instance, +%% this can happen when a queue dead-letters messages to the default exchange without +%% specifying a dead-letter routing key (5). Messages in such cycles (i.e. messages that +%% reach the same queue twice) will be dropped if there was no rejections in the entire cycle. +%% i.e. x-message-ttl (7), x-max-length (6) +%% +%% 6) Message is dead lettered due to queue length limit, and then dropped by the broker as it is +%% republished to the same queue. +dead_letter_routing_key_cycle_max_length(Config) -> + {_Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + Args = ?config(queue_args, Config), + Durable = ?config(queue_durable, Config), + QName = ?config(queue_name, Config), + + DeadLetterArgs = [{<<"x-max-length">>, long, 1}, + {<<"x-dead-letter-exchange">>, longstr, <<>>}], + #'queue.declare_ok'{} = amqp_channel:call(Ch, #'queue.declare'{queue = QName, arguments = DeadLetterArgs ++ Args, durable = Durable}), + + P1 = <<"msg1">>, + P2 = <<"msg2">>, + + %% Publish messages, consume and acknowledge the second one (x-max-length = 1) + publish(Ch, QName, [P1, P2]), + wait_for_messages(Config, [[QName, <<"1">>, <<"1">>, <<"0">>]]), + [DTag] = consume(Ch, QName, [P2]), + consume_empty(Ch, QName), + amqp_channel:cast(Ch, #'basic.ack'{delivery_tag = DTag}), + %% Queue is empty, P1 has not been republished in a loop + wait_for_messages(Config, [[QName, <<"0">>, <<"0">>, <<"0">>]]), + consume_empty(Ch, QName). + +%% 7) Message is dead lettered due to message ttl. Not yet implemented in quorum queues +dead_letter_routing_key_cycle_ttl(Config) -> + {_Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + Args = ?config(queue_args, Config), + Durable = ?config(queue_durable, Config), + QName = ?config(queue_name, Config), + + DeadLetterArgs = [{<<"x-message-ttl">>, long, 1}, + {<<"x-dead-letter-exchange">>, longstr, <<>>}], + #'queue.declare_ok'{} = amqp_channel:call(Ch, #'queue.declare'{queue = QName, arguments = DeadLetterArgs ++ Args, durable = Durable}), + + P1 = <<"msg1">>, + P2 = <<"msg2">>, + + %% Publish messages + publish(Ch, QName, [P1, P2]), + wait_for_messages(Config, [[QName, <<"0">>, <<"0">>, <<"0">>]]), + consume_empty(Ch, QName). + +%% 5) Messages continue to be republished as there are manual rejections +dead_letter_routing_key_cycle_with_reject(Config) -> + {_Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + Args = ?config(queue_args, Config), + Durable = ?config(queue_durable, Config), + QName = ?config(queue_name, Config), + + DeadLetterArgs = [{<<"x-dead-letter-exchange">>, longstr, <<>>}], + #'queue.declare_ok'{} = amqp_channel:call(Ch, #'queue.declare'{queue = QName, arguments = DeadLetterArgs ++ Args, durable = Durable}), + + P = <<"msg1">>, + + %% Publish message + publish(Ch, QName, [P]), + wait_for_messages(Config, [[QName, <<"1">>, <<"1">>, <<"0">>]]), + [DTag] = consume(Ch, QName, [P]), + amqp_channel:cast(Ch, #'basic.nack'{delivery_tag = DTag, + multiple = false, + requeue = false}), + wait_for_messages(Config, [[QName, <<"1">>, <<"1">>, <<"0">>]]), + [DTag1] = consume(Ch, QName, [P]), + amqp_channel:cast(Ch, #'basic.nack'{delivery_tag = DTag1, + multiple = false, + requeue = false}), + %% Message its being republished + wait_for_messages(Config, [[QName, <<"1">>, <<"1">>, <<"0">>]]), + [_] = consume(Ch, QName, [P]). + +%% +%% For any given queue, a DLX can be defined by clients using the queue's arguments, +%% or in the server using policies (8). In the case where both policy and arguments specify a DLX, +%% the one specified in arguments overrules the one specified in policy (9). +%% +%% 8) Use server policies +dead_letter_policy(Config) -> + {_Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + QName = ?config(queue_name, Config), + DLXQName = ?config(queue_name_dlx, Config), + Args = ?config(queue_args, Config), + Durable = ?config(queue_durable, Config), + DLXExchange = ?config(dlx_exchange, Config), + + %% Do not use arguments + #'exchange.declare_ok'{} = amqp_channel:call(Ch, #'exchange.declare'{exchange = DLXExchange}), + #'queue.declare_ok'{} = amqp_channel:call(Ch, #'queue.declare'{queue = QName, arguments = Args, + durable = Durable}), + #'queue.declare_ok'{} = amqp_channel:call(Ch, #'queue.declare'{queue = DLXQName, + durable = Durable}), + #'queue.bind_ok'{} = amqp_channel:call(Ch, #'queue.bind'{queue = DLXQName, + exchange = DLXExchange, + routing_key = DLXQName}), + + P1 = <<"msg1">>, + P2 = <<"msg2">>, + + %% Publish 2 messages + publish(Ch, QName, [P1, P2]), + %% Consume them + wait_for_messages(Config, [[QName, <<"2">>, <<"2">>, <<"0">>]]), + [DTag1, DTag2] = consume(Ch, QName, [P1, P2]), + %% Nack the first one with multiple = false + amqp_channel:cast(Ch, #'basic.nack'{delivery_tag = DTag1, + multiple = false, + requeue = false}), + %% Only one message unack left in the queue + wait_for_messages(Config, [[QName, <<"1">>, <<"0">>, <<"1">>]]), + consume_empty(Ch, QName), + consume_empty(Ch, DLXQName), + + %% Set a policy + ok = rabbit_ct_broker_helpers:set_policy(Config, 0, ?config(policy, Config), QName, + <<"queues">>, + [{<<"dead-letter-exchange">>, DLXExchange}, + {<<"dead-letter-routing-key">>, DLXQName}]), + timer:sleep(1000), + %% Nack the second message + amqp_channel:cast(Ch, #'basic.nack'{delivery_tag = DTag2, + multiple = false, + requeue = false}), + %% Queue is empty + wait_for_messages(Config, [[QName, <<"0">>, <<"0">>, <<"0">>]]), + consume_empty(Ch, QName), + %% Consume the message from the dead letter queue + consume(Ch, DLXQName, [P2]), + consume_empty(Ch, DLXQName). + +%% 9) Argument overrides server policy +dead_letter_override_policy(Config) -> + {_Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + QName = ?config(queue_name, Config), + DLXQName = ?config(queue_name_dlx, Config), + + %% Set a policy, it creates a cycle but message will be republished with the nack. + %% Good enough for this test. + ok = rabbit_ct_broker_helpers:set_policy(Config, 0, ?config(policy, Config), QName, + <<"queues">>, + [{<<"dead-letter-exchange">>, <<>>}, + {<<"dead-letter-routing-key">>, QName}]), + + %% Declare arguments override the policy and set routing queue + declare_dead_letter_queues(Ch, Config, QName, DLXQName), + + P1 = <<"msg1">>, + + publish(Ch, QName, [P1]), + wait_for_messages(Config, [[QName, <<"1">>, <<"1">>, <<"0">>]]), + [DTag1] = consume(Ch, QName, [P1]), + amqp_channel:cast(Ch, #'basic.nack'{delivery_tag = DTag1, + multiple = false, + requeue = false}), + %% Queue is empty + wait_for_messages(Config, [[DLXQName, <<"1">>, <<"1">>, <<"0">>]]), + consume_empty(Ch, QName), + [_] = consume(Ch, DLXQName, [P1]). + +%% 9) Policy is set after have declared a queue with dead letter arguments. Policy will be +%% overridden/ignored. +dead_letter_ignore_policy(Config) -> + {_Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + QName = ?config(queue_name, Config), + DLXQName = ?config(queue_name_dlx, Config), + + declare_dead_letter_queues(Ch, Config, QName, DLXQName), + + %% Set a policy + ok = rabbit_ct_broker_helpers:set_policy(Config, 0, ?config(policy, Config), QName, + <<"queues">>, + [{<<"dead-letter-exchange">>, <<>>}, + {<<"dead-letter-routing-key">>, QName}]), + + P1 = <<"msg1">>, + + publish(Ch, QName, [P1]), + wait_for_messages(Config, [[QName, <<"1">>, <<"1">>, <<"0">>]]), + [DTag1] = consume(Ch, QName, [P1]), + amqp_channel:cast(Ch, #'basic.nack'{delivery_tag = DTag1, + multiple = false, + requeue = false}), + %% Message is in the dead letter queue, original queue is empty + wait_for_messages(Config, [[DLXQName, <<"1">>, <<"1">>, <<"0">>]]), + [_] = consume(Ch, DLXQName, [P1]), + consume_empty(Ch, QName). + +%% +%% HEADERS +%% +%% The dead-lettering process adds an array to the header of each dead-lettered message named +%% x-death (10). This array contains an entry for each dead lettering event containing: +%% queue, reason, time, exchange, routing-keys, count +%% original-expiration (14) (if the message was dead-letterered due to per-message TTL) +%% New entries are prepended to the beginning of the x-death array. +%% Reason is one of the following: rejected (11), expired (12), maxlen (13) +%% +%% 10) and 11) Check all x-death headers, reason rejected +dead_letter_headers(Config) -> + {_Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + QName = ?config(queue_name, Config), + DLXQName = ?config(queue_name_dlx, Config), + declare_dead_letter_queues(Ch, Config, QName, DLXQName), + + %% Publish and nack a message + P1 = <<"msg1">>, + publish(Ch, QName, [P1]), + wait_for_messages(Config, [[QName, <<"1">>, <<"1">>, <<"0">>]]), + [DTag1] = consume(Ch, QName, [P1]), + amqp_channel:cast(Ch, #'basic.nack'{delivery_tag = DTag1, + multiple = false, + requeue = false}), + %% Consume and check headers + wait_for_messages(Config, [[DLXQName, <<"1">>, <<"1">>, <<"0">>]]), + {#'basic.get_ok'{}, #amqp_msg{payload = P1, + props = #'P_basic'{headers = Headers}}} = + amqp_channel:call(Ch, #'basic.get'{queue = DLXQName}), + {array, [{table, Death}]} = rabbit_misc:table_lookup(Headers, <<"x-death">>), + ?assertEqual({longstr, QName}, rabbit_misc:table_lookup(Death, <<"queue">>)), + ?assertEqual({longstr, <<"rejected">>}, rabbit_misc:table_lookup(Death, <<"reason">>)), + ?assertMatch({timestamp, _}, rabbit_misc:table_lookup(Death, <<"time">>)), + ?assertEqual({longstr, <<>>}, rabbit_misc:table_lookup(Death, <<"exchange">>)), + ?assertEqual({long, 1}, rabbit_misc:table_lookup(Death, <<"count">>)), + ?assertEqual({array, [{longstr, QName}]}, rabbit_misc:table_lookup(Death, <<"routing-keys">>)). + +%% 12) Per-queue message ttl has expired +dead_letter_headers_reason_expired(Config) -> + {_Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + QName = ?config(queue_name, Config), + DLXQName = ?config(queue_name_dlx, Config), + declare_dead_letter_queues(Ch, Config, QName, DLXQName, [{<<"x-message-ttl">>, long, 1}]), + + %% Publish a message + P1 = <<"msg1">>, + publish(Ch, QName, [P1]), + %% Consume and check headers + wait_for_messages(Config, [[DLXQName, <<"1">>, <<"1">>, <<"0">>]]), + {#'basic.get_ok'{}, #amqp_msg{payload = P1, + props = #'P_basic'{headers = Headers}}} = + amqp_channel:call(Ch, #'basic.get'{queue = DLXQName}), + {array, [{table, Death}]} = rabbit_misc:table_lookup(Headers, <<"x-death">>), + ?assertEqual({longstr, <<"expired">>}, rabbit_misc:table_lookup(Death, <<"reason">>)), + ?assertMatch(undefined, rabbit_misc:table_lookup(Death, <<"original-expiration">>)). + +%% 14) Per-message TTL has expired, original-expiration is added to x-death array +dead_letter_headers_reason_expired_per_message(Config) -> + {_Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + QName = ?config(queue_name, Config), + DLXQName = ?config(queue_name_dlx, Config), + declare_dead_letter_queues(Ch, Config, QName, DLXQName), + + %% Publish a message + P1 = <<"msg1">>, + amqp_channel:call(Ch, #'basic.publish'{routing_key = QName}, + #amqp_msg{payload = P1, + props = #'P_basic'{expiration = <<"1">>}}), + %% publish another message to ensure the queue performs message expirations + publish(Ch, QName, [<<"msg2">>]), + %% Consume and check headers + wait_for_messages(Config, [[DLXQName, <<"1">>, <<"1">>, <<"0">>]]), + {#'basic.get_ok'{}, #amqp_msg{payload = P1, + props = #'P_basic'{headers = Headers}}} = + amqp_channel:call(Ch, #'basic.get'{queue = DLXQName}), + {array, [{table, Death}]} = rabbit_misc:table_lookup(Headers, <<"x-death">>), + ?assertEqual({longstr, <<"expired">>}, rabbit_misc:table_lookup(Death, <<"reason">>)), + ?assertMatch({longstr, <<"1">>}, rabbit_misc:table_lookup(Death, <<"original-expiration">>)). + +%% 13) Message expired with maxlen reason +dead_letter_headers_reason_maxlen(Config) -> + {_Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + QName = ?config(queue_name, Config), + DLXQName = ?config(queue_name_dlx, Config), + declare_dead_letter_queues(Ch, Config, QName, DLXQName, [{<<"x-max-length">>, long, 1}]), + + P1 = <<"msg1">>, + P2 = <<"msg2">>, + publish(Ch, QName, [P1, P2]), + %% Consume and check reason header + wait_for_messages(Config, [[DLXQName, <<"1">>, <<"1">>, <<"0">>]]), + {#'basic.get_ok'{}, #amqp_msg{payload = P1, + props = #'P_basic'{headers = Headers}}} = + amqp_channel:call(Ch, #'basic.get'{queue = DLXQName}), + {array, [{table, Death}]} = rabbit_misc:table_lookup(Headers, <<"x-death">>), + ?assertEqual({longstr, <<"maxlen">>}, rabbit_misc:table_lookup(Death, <<"reason">>)). + +%% In case x-death already contains an entry with the same queue and dead lettering reason, +%% its count field will be incremented and it will be moved to the beginning of the array +dead_letter_headers_cycle(Config) -> + {_Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + Args = ?config(queue_args, Config), + Durable = ?config(queue_durable, Config), + QName = ?config(queue_name, Config), + + DeadLetterArgs = [{<<"x-dead-letter-exchange">>, longstr, <<>>}], + #'queue.declare_ok'{} = amqp_channel:call(Ch, #'queue.declare'{queue = QName, arguments = DeadLetterArgs ++ Args, durable = Durable}), + + P = <<"msg1">>, + + %% Publish message + publish(Ch, QName, [P]), + wait_for_messages(Config, [[QName, <<"1">>, <<"1">>, <<"0">>]]), + [DTag] = consume(Ch, QName, [P]), + amqp_channel:cast(Ch, #'basic.nack'{delivery_tag = DTag, + multiple = false, + requeue = false}), + wait_for_messages(Config, [[QName, <<"1">>, <<"1">>, <<"0">>]]), + {#'basic.get_ok'{delivery_tag = DTag1}, #amqp_msg{payload = P, + props = #'P_basic'{headers = Headers1}}} = + amqp_channel:call(Ch, #'basic.get'{queue = QName}), + {array, [{table, Death1}]} = rabbit_misc:table_lookup(Headers1, <<"x-death">>), + ?assertEqual({long, 1}, rabbit_misc:table_lookup(Death1, <<"count">>)), + + amqp_channel:cast(Ch, #'basic.nack'{delivery_tag = DTag1, + multiple = false, + requeue = false}), + %% Message its being republished + wait_for_messages(Config, [[QName, <<"1">>, <<"1">>, <<"0">>]]), + {#'basic.get_ok'{}, #amqp_msg{payload = P, + props = #'P_basic'{headers = Headers2}}} = + amqp_channel:call(Ch, #'basic.get'{queue = QName}), + {array, [{table, Death2}]} = rabbit_misc:table_lookup(Headers2, <<"x-death">>), + ?assertEqual({long, 2}, rabbit_misc:table_lookup(Death2, <<"count">>)). + +%% Dead-lettering a message modifies its headers: +%% the exchange name is replaced with that of the latest dead-letter exchange, +%% the routing key may be replaced with that specified in a queue performing dead lettering, +%% if the above happens, the CC header will also be removed (15) and +%% the BCC header will be removed as per Sender-selected distribution (16) +%% +%% CC header is kept if no dead lettering routing key is provided +dead_letter_headers_CC(Config) -> + {_Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + QName = ?config(queue_name, Config), + DLXQName = ?config(queue_name_dlx, Config), + Args = ?config(queue_args, Config), + Durable = ?config(queue_durable, Config), + DLXExchange = ?config(dlx_exchange, Config), + + %% Do not use a specific key for dead lettering, the CC header is passed + DeadLetterArgs = [{<<"x-dead-letter-exchange">>, longstr, DLXExchange}], + #'exchange.declare_ok'{} = amqp_channel:call(Ch, #'exchange.declare'{exchange = DLXExchange}), + #'queue.declare_ok'{} = amqp_channel:call(Ch, #'queue.declare'{queue = QName, arguments = DeadLetterArgs ++ Args, durable = Durable}), + #'queue.declare_ok'{} = amqp_channel:call(Ch, #'queue.declare'{queue = DLXQName, durable = Durable}), + #'queue.bind_ok'{} = amqp_channel:call(Ch, #'queue.bind'{queue = DLXQName, + exchange = DLXExchange, + routing_key = DLXQName}), + + P1 = <<"msg1">>, + CCHeader = {<<"CC">>, array, [{longstr, DLXQName}]}, + publish(Ch, QName, [P1], [CCHeader]), + wait_for_messages(Config, [[QName, <<"1">>, <<"1">>, <<"0">>]]), + %% Message is published to both queues because of CC header and DLX queue bound to both + %% exchanges + {#'basic.get_ok'{delivery_tag = DTag1}, #amqp_msg{payload = P1, + props = #'P_basic'{headers = Headers1}}} = + amqp_channel:call(Ch, #'basic.get'{queue = QName}), + {#'basic.get_ok'{}, #amqp_msg{payload = P1, + props = #'P_basic'{headers = Headers2}}} = + amqp_channel:call(Ch, #'basic.get'{queue = DLXQName}), + %% We check the headers to ensure no dead lettering has happened + ?assertEqual(undefined, rabbit_misc:table_lookup(Headers1, <<"x-death">>)), + ?assertEqual(undefined, rabbit_misc:table_lookup(Headers2, <<"x-death">>)), + + %% Nack the message so it now gets dead lettered + amqp_channel:cast(Ch, #'basic.nack'{delivery_tag = DTag1, + multiple = false, + requeue = false}), + wait_for_messages(Config, [[DLXQName, <<"2">>, <<"1">>, <<"1">>]]), + {#'basic.get_ok'{}, #amqp_msg{payload = P1, + props = #'P_basic'{headers = Headers3}}} = + amqp_channel:call(Ch, #'basic.get'{queue = DLXQName}), + consume_empty(Ch, QName), + ?assertEqual({array, [{longstr, DLXQName}]}, rabbit_misc:table_lookup(Headers3, <<"CC">>)), + ?assertMatch({array, _}, rabbit_misc:table_lookup(Headers3, <<"x-death">>)). + +%% 15) CC header is removed when routing key is specified +dead_letter_headers_CC_with_routing_key(Config) -> + {_Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + QName = ?config(queue_name, Config), + DLXQName = ?config(queue_name_dlx, Config), + Args = ?config(queue_args, Config), + Durable = ?config(queue_durable, Config), + DLXExchange = ?config(dlx_exchange, Config), + + %% Do not use a specific key for dead lettering, the CC header is passed + DeadLetterArgs = [{<<"x-dead-letter-routing-key">>, longstr, DLXQName}, + {<<"x-dead-letter-exchange">>, longstr, DLXExchange}], + #'exchange.declare_ok'{} = amqp_channel:call(Ch, #'exchange.declare'{exchange = DLXExchange}), + #'queue.declare_ok'{} = amqp_channel:call(Ch, #'queue.declare'{queue = QName, arguments = DeadLetterArgs ++ Args, durable = Durable}), + #'queue.declare_ok'{} = amqp_channel:call(Ch, #'queue.declare'{queue = DLXQName, durable = Durable}), + #'queue.bind_ok'{} = amqp_channel:call(Ch, #'queue.bind'{queue = DLXQName, + exchange = DLXExchange, + routing_key = DLXQName}), + + P1 = <<"msg1">>, + CCHeader = {<<"CC">>, array, [{longstr, DLXQName}]}, + publish(Ch, QName, [P1], [CCHeader]), + wait_for_messages(Config, [[QName, <<"1">>, <<"1">>, <<"0">>]]), + %% Message is published to both queues because of CC header and DLX queue bound to both + %% exchanges + {#'basic.get_ok'{delivery_tag = DTag1}, #amqp_msg{payload = P1, + props = #'P_basic'{headers = Headers1}}} = + amqp_channel:call(Ch, #'basic.get'{queue = QName}), + {#'basic.get_ok'{}, #amqp_msg{payload = P1, + props = #'P_basic'{headers = Headers2}}} = + amqp_channel:call(Ch, #'basic.get'{queue = DLXQName}), + %% We check the headers to ensure no dead lettering has happened + ?assertEqual(undefined, rabbit_misc:table_lookup(Headers1, <<"x-death">>)), + ?assertEqual(undefined, rabbit_misc:table_lookup(Headers2, <<"x-death">>)), + + %% Nack the message so it now gets dead lettered + amqp_channel:cast(Ch, #'basic.nack'{delivery_tag = DTag1, + multiple = false, + requeue = false}), + wait_for_messages(Config, [[DLXQName, <<"2">>, <<"1">>, <<"1">>]]), + {#'basic.get_ok'{}, #amqp_msg{payload = P1, + props = #'P_basic'{headers = Headers3}}} = + amqp_channel:call(Ch, #'basic.get'{queue = DLXQName}), + consume_empty(Ch, QName), + ?assertEqual(undefined, rabbit_misc:table_lookup(Headers3, <<"CC">>)), + ?assertMatch({array, _}, rabbit_misc:table_lookup(Headers3, <<"x-death">>)). + +%% 16) the BCC header will always be removed +dead_letter_headers_BCC(Config) -> + {_Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + QName = ?config(queue_name, Config), + DLXQName = ?config(queue_name_dlx, Config), + Args = ?config(queue_args, Config), + Durable = ?config(queue_durable, Config), + DLXExchange = ?config(dlx_exchange, Config), + + %% Do not use a specific key for dead lettering + DeadLetterArgs = [{<<"x-dead-letter-exchange">>, longstr, DLXExchange}], + #'exchange.declare_ok'{} = amqp_channel:call(Ch, #'exchange.declare'{exchange = DLXExchange}), + #'queue.declare_ok'{} = amqp_channel:call(Ch, #'queue.declare'{queue = QName, arguments = DeadLetterArgs ++ Args, durable = Durable}), + #'queue.declare_ok'{} = amqp_channel:call(Ch, #'queue.declare'{queue = DLXQName, durable = Durable}), + #'queue.bind_ok'{} = amqp_channel:call(Ch, #'queue.bind'{queue = DLXQName, + exchange = DLXExchange, + routing_key = DLXQName}), + + P1 = <<"msg1">>, + BCCHeader = {<<"BCC">>, array, [{longstr, DLXQName}]}, + publish(Ch, QName, [P1], [BCCHeader]), + %% Message is published to both queues because of BCC header and DLX queue bound to both + %% exchanges + wait_for_messages(Config, [[QName, <<"1">>, <<"1">>, <<"0">>]]), + {#'basic.get_ok'{delivery_tag = DTag1}, #amqp_msg{payload = P1, + props = #'P_basic'{headers = Headers1}}} = + amqp_channel:call(Ch, #'basic.get'{queue = QName}), + {#'basic.get_ok'{}, #amqp_msg{payload = P1, + props = #'P_basic'{headers = Headers2}}} = + amqp_channel:call(Ch, #'basic.get'{queue = DLXQName}), + %% We check the headers to ensure no dead lettering has happened + ?assertEqual(undefined, rabbit_misc:table_lookup(Headers1, <<"x-death">>)), + ?assertEqual(undefined, rabbit_misc:table_lookup(Headers2, <<"x-death">>)), + + %% Nack the message so it now gets dead lettered + amqp_channel:cast(Ch, #'basic.nack'{delivery_tag = DTag1, + multiple = false, + requeue = false}), + wait_for_messages(Config, [[DLXQName, <<"2">>, <<"1">>, <<"1">>]]), + {#'basic.get_ok'{}, #amqp_msg{payload = P1, + props = #'P_basic'{headers = Headers3}}} = + amqp_channel:call(Ch, #'basic.get'{queue = DLXQName}), + consume_empty(Ch, QName), + ?assertEqual(undefined, rabbit_misc:table_lookup(Headers3, <<"BCC">>)), + ?assertMatch({array, _}, rabbit_misc:table_lookup(Headers3, <<"x-death">>)). + + +%% Three top-level headers are added for the very first dead-lettering event. +%% They are +%% x-first-death-reason, x-first-death-queue, x-first-death-exchange +%% They have the same values as the reason, queue, and exchange fields of the +%% original +%% dead lettering event. Once added, these headers are never modified. +dead_letter_headers_first_death(Config) -> + {_Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + QName = ?config(queue_name, Config), + DLXQName = ?config(queue_name_dlx, Config), + Args = ?config(queue_args, Config), + Durable = ?config(queue_durable, Config), + DLXExchange = ?config(dlx_exchange, Config), + + %% Let's create a small dead-lettering loop QName -> DLXQName -> QName + DeadLetterArgs = [{<<"x-dead-letter-routing-key">>, longstr, DLXQName}, + {<<"x-dead-letter-exchange">>, longstr, DLXExchange}], + DLXDeadLetterArgs = [{<<"x-dead-letter-routing-key">>, longstr, QName}, + {<<"x-dead-letter-exchange">>, longstr, <<>>}], + #'exchange.declare_ok'{} = amqp_channel:call(Ch, #'exchange.declare'{exchange = DLXExchange}), + #'queue.declare_ok'{} = amqp_channel:call(Ch, #'queue.declare'{queue = QName, arguments = DeadLetterArgs ++ Args, durable = Durable}), + #'queue.declare_ok'{} = amqp_channel:call(Ch, #'queue.declare'{queue = DLXQName, durable = Durable, arguments = DLXDeadLetterArgs}), + #'queue.bind_ok'{} = amqp_channel:call(Ch, #'queue.bind'{queue = DLXQName, + exchange = DLXExchange, + routing_key = DLXQName}), + + + %% Publish and nack a message + P1 = <<"msg1">>, + publish(Ch, QName, [P1]), + wait_for_messages(Config, [[QName, <<"1">>, <<"1">>, <<"0">>]]), + [DTag1] = consume(Ch, QName, [P1]), + amqp_channel:cast(Ch, #'basic.nack'{delivery_tag = DTag1, + multiple = false, + requeue = false}), + %% Consume and check headers + wait_for_messages(Config, [[DLXQName, <<"1">>, <<"1">>, <<"0">>]]), + {#'basic.get_ok'{delivery_tag = DTag2}, #amqp_msg{payload = P1, + props = #'P_basic'{headers = Headers}}} = + amqp_channel:call(Ch, #'basic.get'{queue = DLXQName}), + ?assertEqual({longstr, <<"rejected">>}, + rabbit_misc:table_lookup(Headers, <<"x-first-death-reason">>)), + ?assertEqual({longstr, QName}, + rabbit_misc:table_lookup(Headers, <<"x-first-death-queue">>)), + ?assertEqual({longstr, <<>>}, + rabbit_misc:table_lookup(Headers, <<"x-first-death-exchange">>)), + %% Nack the message again so it gets dead lettered to the initial queue. x-first-death + %% headers should not change + amqp_channel:cast(Ch, #'basic.nack'{delivery_tag = DTag2, + multiple = false, + requeue = false}), + wait_for_messages(Config, [[QName, <<"1">>, <<"1">>, <<"0">>]]), + {#'basic.get_ok'{}, #amqp_msg{payload = P1, + props = #'P_basic'{headers = Headers2}}} = + amqp_channel:call(Ch, #'basic.get'{queue = QName}), + ?assertEqual({longstr, <<"rejected">>}, + rabbit_misc:table_lookup(Headers2, <<"x-first-death-reason">>)), + ?assertEqual({longstr, QName}, + rabbit_misc:table_lookup(Headers2, <<"x-first-death-queue">>)), + ?assertEqual({longstr, <<>>}, + rabbit_misc:table_lookup(Headers2, <<"x-first-death-exchange">>)). + +%%%%%%%%%%%%%%%%%%%%%%%% +%% Test helpers +%%%%%%%%%%%%%%%%%%%%%%%% +declare_dead_letter_queues(Ch, Config, QName, DLXQName) -> + declare_dead_letter_queues(Ch, Config, QName, DLXQName, []). + +declare_dead_letter_queues(Ch, Config, QName, DLXQName, ExtraArgs) -> + Args = ?config(queue_args, Config), + Durable = ?config(queue_durable, Config), + DLXExchange = ?config(dlx_exchange, Config), + + %% Declare DLX exchange + #'exchange.declare_ok'{} = amqp_channel:call(Ch, #'exchange.declare'{exchange = DLXExchange}), + + %% Declare queue + DeadLetterArgs = [{<<"x-dead-letter-exchange">>, longstr, DLXExchange}, + {<<"x-dead-letter-routing-key">>, longstr, DLXQName}], + #'queue.declare_ok'{} = amqp_channel:call(Ch, #'queue.declare'{queue = QName, arguments = DeadLetterArgs ++ Args ++ ExtraArgs, durable = Durable}), + + %% Declare and bind DLX queue + #'queue.declare_ok'{} = amqp_channel:call(Ch, #'queue.declare'{queue = DLXQName, durable = Durable}), + #'queue.bind_ok'{} = amqp_channel:call(Ch, #'queue.bind'{queue = DLXQName, + exchange = DLXExchange, + routing_key = DLXQName}). + +publish(Ch, QName, Payloads) -> + [amqp_channel:call(Ch, #'basic.publish'{routing_key = QName}, #amqp_msg{payload = Payload}) + || Payload <- Payloads]. + +publish(Ch, QName, Payloads, Headers) -> + [amqp_channel:call(Ch, #'basic.publish'{routing_key = QName}, + #amqp_msg{payload = Payload, + props = #'P_basic'{headers = Headers}}) + || Payload <- Payloads]. + +consume(Ch, QName, Payloads) -> + [begin + {#'basic.get_ok'{delivery_tag = DTag}, #amqp_msg{payload = Payload}} = + amqp_channel:call(Ch, #'basic.get'{queue = QName}), + DTag + end || Payload <- Payloads]. + +consume_empty(Ch, QName) -> + #'basic.get_empty'{} = amqp_channel:call(Ch, #'basic.get'{queue = QName}). + +sync_mirrors(QName, Config) -> + case ?config(is_mirrored, Config) of + true -> + rabbit_ct_broker_helpers:rabbitmqctl(Config, 0, [<<"sync_queue">>, QName]); + _ -> ok + end. diff --git a/deps/rabbit/test/definition_import_SUITE.erl b/deps/rabbit/test/definition_import_SUITE.erl new file mode 100644 index 0000000000..ac0c18da99 --- /dev/null +++ b/deps/rabbit/test/definition_import_SUITE.erl @@ -0,0 +1,257 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(definition_import_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("rabbit_common/include/rabbit.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-compile(export_all). + +all() -> + [ + {group, boot_time_import}, + {group, roundtrip}, + {group, import_on_a_running_node} + ]. + +groups() -> + [ + {import_on_a_running_node, [], [ + %% Note: to make it easier to see which case failed, + %% these are intentionally not folded into a single case. + %% If generation becomes an alternative worth considering for these tests, + %% we'll just add a case that drives PropEr. + import_case1, + import_case2, + import_case3, + import_case4, + import_case5, + import_case6, + import_case7, + import_case8, + import_case9, + import_case10, + import_case11, + import_case12, + import_case13 + ]}, + {boot_time_import, [], [ + import_on_a_booting_node + ]}, + + {roundtrip, [], [ + export_import_round_trip_case1, + export_import_round_trip_case2 + ]} + ]. + +%% ------------------------------------------------------------------- +%% Test suite setup/teardown. +%% ------------------------------------------------------------------- + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + inets:start(), + Config. +end_per_suite(Config) -> + Config. + +init_per_group(boot_time_import = Group, Config) -> + CasePath = filename:join(?config(data_dir, Config), "case5.json"), + Config1 = rabbit_ct_helpers:set_config(Config, [ + {rmq_nodename_suffix, Group}, + {rmq_nodes_count, 1} + ]), + Config2 = rabbit_ct_helpers:merge_app_env(Config1, + {rabbit, [ + {load_definitions, CasePath} + ]}), + rabbit_ct_helpers:run_setup_steps(Config2, rabbit_ct_broker_helpers:setup_steps()); +init_per_group(Group, Config) -> + Config1 = rabbit_ct_helpers:set_config(Config, [ + {rmq_nodename_suffix, Group} + ]), + rabbit_ct_helpers:run_setup_steps(Config1, rabbit_ct_broker_helpers:setup_steps()). + +end_per_group(_, Config) -> + rabbit_ct_helpers:run_teardown_steps(Config, rabbit_ct_broker_helpers:teardown_steps()). + +init_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_started(Config, Testcase). + +end_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_finished(Config, Testcase). + +%% +%% Tests +%% + +import_case1(Config) -> import_file_case(Config, "case1"). +import_case2(Config) -> import_file_case(Config, "case2"). +import_case3(Config) -> import_file_case(Config, "case3"). +import_case4(Config) -> import_file_case(Config, "case4"). +import_case6(Config) -> import_file_case(Config, "case6"). +import_case7(Config) -> import_file_case(Config, "case7"). +import_case8(Config) -> import_file_case(Config, "case8"). + +import_case9(Config) -> import_from_directory_case(Config, "case9"). + +import_case10(Config) -> import_from_directory_case_fails(Config, "case10"). + +import_case5(Config) -> + import_file_case(Config, "case5"), + ?assertEqual(rabbit_ct_broker_helpers:rpc(Config, 0, + rabbit_runtime_parameters, value_global, + [mqtt_port_to_vhost_mapping]), + %% expect a proplist, see rabbitmq/rabbitmq-management#528 + [{<<"1883">>,<<"/">>}, + {<<"1884">>,<<"vhost2">>}]). + +import_case11(Config) -> import_file_case(Config, "case11"). +import_case12(Config) -> import_invalid_file_case(Config, "failing_case12"). + +import_case13(Config) -> + case rabbit_ct_broker_helpers:enable_feature_flag(Config, quorum_queue) of + ok -> + import_file_case(Config, "case13"), + VHost = <<"/">>, + QueueName = <<"definitions.import.case13.qq.1">>, + QueueIsImported = + fun () -> + case queue_lookup(Config, VHost, QueueName) of + {ok, _} -> true; + _ -> false + end + end, + rabbit_ct_helpers:await_condition(QueueIsImported, 20000), + {ok, Q} = queue_lookup(Config, VHost, QueueName), + + %% see rabbitmq/rabbitmq-server#2400, rabbitmq/rabbitmq-server#2426 + ?assert(amqqueue:is_quorum(Q)), + ?assertEqual([{<<"x-max-length">>, long, 991}, + {<<"x-queue-type">>, longstr, <<"quorum">>}], + amqqueue:get_arguments(Q)); + Skip -> + Skip + end. + +export_import_round_trip_case1(Config) -> + %% case 6 has runtime parameters that do not depend on any plugins + import_file_case(Config, "case6"), + Defs = export(Config), + import_raw(Config, rabbit_json:encode(Defs)). + +export_import_round_trip_case2(Config) -> + import_file_case(Config, "case9", "case9a"), + Defs = export(Config), + import_parsed(Config, Defs). + +import_on_a_booting_node(Config) -> + %% see case5.json + VHost = <<"vhost2">>, + %% verify that vhost2 eventually starts + case rabbit_ct_broker_helpers:rpc(Config, 0, rabbit_vhost, await_running_on_all_nodes, [VHost, 3000]) of + ok -> ok; + {error, timeout} -> ct:fail("virtual host ~p was not imported on boot", [VHost]) + end. + +%% +%% Implementation +%% + +import_file_case(Config, CaseName) -> + CasePath = filename:join([ + ?config(data_dir, Config), + CaseName ++ ".json" + ]), + rabbit_ct_broker_helpers:rpc(Config, 0, ?MODULE, run_import_case, [CasePath]), + ok. + +import_file_case(Config, Subdirectory, CaseName) -> + CasePath = filename:join([ + ?config(data_dir, Config), + Subdirectory, + CaseName ++ ".json" + ]), + rabbit_ct_broker_helpers:rpc(Config, 0, ?MODULE, run_import_case, [CasePath]), + ok. + +import_invalid_file_case(Config, CaseName) -> + CasePath = filename:join(?config(data_dir, Config), CaseName ++ ".json"), + rabbit_ct_broker_helpers:rpc(Config, 0, ?MODULE, run_invalid_import_case, [CasePath]), + ok. + +import_from_directory_case(Config, CaseName) -> + import_from_directory_case_expect(Config, CaseName, ok). + +import_from_directory_case_fails(Config, CaseName) -> + import_from_directory_case_expect(Config, CaseName, error). + +import_from_directory_case_expect(Config, CaseName, Expected) -> + CasePath = filename:join(?config(data_dir, Config), CaseName), + ?assert(filelib:is_dir(CasePath)), + rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, run_directory_import_case, + [CasePath, Expected]), + ok. + +import_raw(Config, Body) -> + case rabbit_ct_broker_helpers:rpc(Config, 0, rabbit_definitions, import_raw, [Body]) of + ok -> ok; + {error, E} -> + ct:pal("Import of JSON definitions ~p failed: ~p~n", [Body, E]), + ct:fail({failure, Body, E}) + end. + +import_parsed(Config, Body) -> + case rabbit_ct_broker_helpers:rpc(Config, 0, rabbit_definitions, import_parsed, [Body]) of + ok -> ok; + {error, E} -> + ct:pal("Import of parsed definitions ~p failed: ~p~n", [Body, E]), + ct:fail({failure, Body, E}) + end. + +export(Config) -> + rabbit_ct_broker_helpers:rpc(Config, 0, ?MODULE, run_export, []). + +run_export() -> + rabbit_definitions:all_definitions(). + +run_directory_import_case(Path, Expected) -> + ct:pal("Will load definitions from files under ~p~n", [Path]), + Result = rabbit_definitions:maybe_load_definitions_from(true, Path), + case Expected of + ok -> + ok = Result; + error -> + ?assertMatch({error, {failed_to_import_definitions, _, _}}, Result) + end. + +run_import_case(Path) -> + {ok, Body} = file:read_file(Path), + ct:pal("Successfully loaded a definition to import from ~p~n", [Path]), + case rabbit_definitions:import_raw(Body) of + ok -> ok; + {error, E} -> + ct:pal("Import case ~p failed: ~p~n", [Path, E]), + ct:fail({failure, Path, E}) + end. + +run_invalid_import_case(Path) -> + {ok, Body} = file:read_file(Path), + ct:pal("Successfully loaded a definition to import from ~p~n", [Path]), + case rabbit_definitions:import_raw(Body) of + ok -> + ct:pal("Expected import case ~p to fail~n", [Path]), + ct:fail({failure, Path}); + {error, _E} -> ok + end. + +queue_lookup(Config, VHost, Name) -> + rabbit_ct_broker_helpers:rpc(Config, 0, rabbit_amqqueue, lookup, [rabbit_misc:r(VHost, queue, Name)]). diff --git a/deps/rabbit/test/definition_import_SUITE_data/case1.json b/deps/rabbit/test/definition_import_SUITE_data/case1.json new file mode 100644 index 0000000000..b0785a5214 --- /dev/null +++ b/deps/rabbit/test/definition_import_SUITE_data/case1.json @@ -0,0 +1,99 @@ +{ + "rabbit_version": "3.6.9", + "users": [ + { + "name": "project_admin", + "password_hash": "A0EX\/2hiwrIDKFS+nEqwbCGcVxwEkDBFF3mBfkNW53KFFk64", + "hashing_algorithm": "rabbit_password_hashing_sha256", + "tags": "" + } + ], + "vhosts": [ + { + "name": "\/" + } + ], + "permissions": [ + { + "user": "project_admin", + "vhost": "\/", + "configure": ".*", + "write": ".*", + "read": ".*" + } + ], + "policies": [ + { + "vhost": "\/", + "name": "nd-ns", + "pattern": "^project-nd-ns-", + "apply-to": "queues", + "definition": { + "expires": 120000, + "max-length": 10000 + }, + "priority": 1 + }, + { + "vhost": "\/", + "name": "nd-s", + "pattern": "^project-nd-s-", + "apply-to": "queues", + "definition": { + "expires": 1800000, + "max-length": 50000 + }, + "priority": 1 + }, + { + "vhost": "\/", + "name": "d-ns", + "pattern": "^project-d-ns-", + "apply-to": "queues", + "definition": { + "ha-mode": "exactly", + "ha-params": 3, + "ha-sync-mode": "automatic", + "expires": 604800000, + "ha-sync-batch-size": 100, + "queue-mode": "lazy" + }, + "priority": 1 + }, + { + "vhost": "\/", + "name": "d-s", + "pattern": "^project-d-s-", + "apply-to": "queues", + "definition": { + "ha-mode": "exactly", + "ha-params": 3, + "ha-sync-mode": "automatic", + "expires": 604800000, + "queue-master-locator": "min-masters", + "ha-sync-batch-size": 100, + "queue-mode": "lazy" + }, + "priority": 1 + } + ], + "queues": [ + + ], + "exchanges": [ + { + "name": "project.topic.default", + "vhost": "\/", + "type": "topic", + "durable": true, + "auto_delete": false, + "internal": false, + "arguments": { + + } + } + ], + "bindings": [ + + ] +}
\ No newline at end of file diff --git a/deps/rabbit/test/definition_import_SUITE_data/case10/case10a.json b/deps/rabbit/test/definition_import_SUITE_data/case10/case10a.json new file mode 100644 index 0000000000..1eec5ccb9e --- /dev/null +++ b/deps/rabbit/test/definition_import_SUITE_data/case10/case10a.json @@ -0,0 +1,67 @@ +{ + "rabbit_version": "3.7.13", + "users": [ + { + "name": "bunny_reader", + "password_hash": "ExmGdjBTmQEPxcW2z+dsOuPvjFbTBiYQgMByzfpE/IIXplYG", + "hashing_algorithm": "rabbit_password_hashing_sha256", + "tags": "" + } + ], + "vhosts": [ + { + "name": "langohr_testbed" + }, + { + "name": "bunny_testbed" + }, + { + "name": "/" + } + ], + "permissions": [ + { + "user": "bunny_reader", + "vhost": "bunny_testbed", + "configure": "^---$", + "write": "^---$", + "read": ".*" + } + ], + "topic_permissions": [], + "parameters": [ + { + "component": "vhost-limits", + "name": "limits", + "value": { + "max-connections": 14000 + }, + "vhost": "/" + } + ], + "global_parameters": [ + { + "name": "cluster_name", + "value": "rabbit@localhost" + } + ], + "policies": [], + "queues": [ + { + "name": "bunny.basic_consume0.1364356981103202", + "vhost": "bunny_testbed", + "durable": false, + "auto_delete": true, + "arguments": {} + }, + { + "name": "bunny.basic_consume0.1364356981103202", + "vhost": "bunny_testbed", + "durable": true, + "auto_delete": true, + "arguments": {} + } + ], + "exchanges": [], + "bindings": [] +} diff --git a/deps/rabbit/test/definition_import_SUITE_data/case10/case10b.json b/deps/rabbit/test/definition_import_SUITE_data/case10/case10b.json new file mode 100644 index 0000000000..9eb48e341e --- /dev/null +++ b/deps/rabbit/test/definition_import_SUITE_data/case10/case10b.json @@ -0,0 +1,595 @@ +{ + "rabbit_version": "3.7.13", + "users": [ + { + "name": "langohr", + "password_hash": "7p9PXlsYs92NlHSdNgPoDXmN77NqeGpzCTHpElq/wPS1eAEd", + "hashing_algorithm": "rabbit_password_hashing_sha256", + "tags": "" + }, + { + "name": "bunny_reader", + "password_hash": "ExmGdjBTmQEPxcW2z+dsOuPvjFbTBiYQgMByzfpE/IIXplYG", + "hashing_algorithm": "rabbit_password_hashing_sha256", + "tags": "" + }, + { + "name": "bunny_gem", + "password_hash": "8HH7uxmZS3FDldlYmHpFEE5+gWaeQaim8qpWIHkmNxuQK8xO", + "hashing_algorithm": "rabbit_password_hashing_sha256", + "tags": "" + }, + { + "name": "guest2", + "password_hash": "E04A7cvvsaDJBezc3Sc2jCnywe9oS4DX18qFe4dwkjIr26gf", + "hashing_algorithm": "rabbit_password_hashing_sha256", + "tags": "monitoring" + }, + { + "name": "guest", + "password_hash": "CPCbkNAHXgQ7vmrqwP9e7RWQsE8U2DqN7JA4ggS50c4LwDda", + "hashing_algorithm": "rabbit_password_hashing_sha256", + "tags": "administrator" + }, + { + "name": "temp-user", + "password_hash": "CfUQkDeOYDrPkACDCjoF5zySbsXPIoMgNfv7FWfEpVFGegnL", + "hashing_algorithm": "rabbit_password_hashing_sha256", + "tags": "management" + } + ], + "vhosts": [ + { + "name": "langohr_testbed" + }, + { + "name": "bunny_testbed" + }, + { + "name": "/" + }, + { + "name": "vhost3" + } + ], + "permissions": [ + { + "user": "bunny_reader", + "vhost": "bunny_testbed", + "configure": "^---$", + "write": "^---$", + "read": ".*" + }, + { + "user": "bunny_gem", + "vhost": "bunny_testbed", + "configure": ".*", + "write": ".*", + "read": ".*" + }, + { + "user": "guest", + "vhost": "/", + "configure": ".*", + "write": ".*", + "read": ".*" + }, + { + "user": "langohr", + "vhost": "langohr_testbed", + "configure": ".*", + "write": ".*", + "read": ".*" + }, + { + "user": "guest", + "vhost": "bunny_testbed", + "configure": ".*", + "write": ".*", + "read": ".*" + }, + { + "user": "guest", + "vhost": "langohr_testbed", + "configure": ".*", + "write": ".*", + "read": ".*" + }, + { + "user": "guest", + "vhost": "vhost3", + "configure": ".*", + "write": ".*", + "read": ".*" + }, + { + "user": "langohr", + "vhost": "/", + "configure": ".*", + "write": ".*", + "read": ".*" + }, + { + "user": "temp-user", + "vhost": "/", + "configure": ".*", + "write": ".*", + "read": ".*" + } + ], + "topic_permissions": [], + "parameters": [ + { + "value": { + "expires": 3600000, + "uri": "amqp://localhost:5673" + }, + "vhost": "/", + "component": "federation-upstream", + "name": "up-hare" + }, + { + "value": { + "max-connections": 2000 + }, + "vhost": "/", + "component": "vhost-limits", + "name": "limits" + } + ], + "global_parameters": [ + { + "name": "cluster_name", + "value": "rabbit@localhost" + } + ], + "policies": [], + "queues": [ + { + "name": "bunny.basic_consume0.7103611911099639", + "vhost": "bunny_testbed", + "durable": false, + "auto_delete": true, + "arguments": {} + }, + { + "name": "bunny.basic_consume0.6091120557781405", + "vhost": "bunny_testbed", + "durable": false, + "auto_delete": true, + "arguments": {} + }, + { + "name": "bunny.basic_consume0.8661861002262826", + "vhost": "bunny_testbed", + "durable": false, + "auto_delete": true, + "arguments": {} + }, + { + "name": "bunny.basic_consume0.3682573609392056", + "vhost": "bunny_testbed", + "durable": false, + "auto_delete": true, + "arguments": {} + }, + { + "name": "bunny.basic_consume0.14855593896585362", + "vhost": "bunny_testbed", + "durable": false, + "auto_delete": true, + "arguments": {} + }, + { + "name": "bunny.basic_consume0.9534242141484872", + "vhost": "bunny_testbed", + "durable": false, + "auto_delete": true, + "arguments": {} + }, + { + "name": "bunny.basic_consume0.9434723539955824", + "vhost": "bunny_testbed", + "durable": false, + "auto_delete": true, + "arguments": {} + }, + { + "name": "bunny.basic_consume0.12235844522013617", + "vhost": "bunny_testbed", + "durable": false, + "auto_delete": true, + "arguments": {} + }, + { + "name": "bunny.basic_consume0.8370997977912426", + "vhost": "bunny_testbed", + "durable": false, + "auto_delete": true, + "arguments": {} + }, + { + "name": "bunny.basic_consume0.4548488370639835", + "vhost": "bunny_testbed", + "durable": false, + "auto_delete": true, + "arguments": {} + }, + { + "name": "bunny.basic_consume0.2289868670635532", + "vhost": "bunny_testbed", + "durable": false, + "auto_delete": true, + "arguments": {} + }, + { + "name": "bunny.basic_consume0.00797124769641977", + "vhost": "bunny_testbed", + "durable": false, + "auto_delete": true, + "arguments": {} + }, + { + "name": "amq.gen-xddEPq9wHSNZKQbPK8pi3A", + "vhost": "bunny_testbed", + "durable": false, + "auto_delete": false, + "arguments": {} + }, + { + "name": "bunny.basic_consume0.5195700828676673", + "vhost": "bunny_testbed", + "durable": false, + "auto_delete": true, + "arguments": {} + }, + { + "name": "bunny.basic_consume0.3071859764599716", + "vhost": "bunny_testbed", + "durable": false, + "auto_delete": true, + "arguments": {} + }, + { + "name": "return", + "vhost": "/", + "durable": false, + "auto_delete": true, + "arguments": {} + }, + { + "name": "q1", + "vhost": "/", + "durable": true, + "auto_delete": false, + "arguments": {} + }, + { + "name": "declareArgs-deliveries-dead-letter", + "vhost": "/", + "durable": false, + "auto_delete": true, + "arguments": {} + }, + { + "name": "test.rabbitmq-basic-nack", + "vhost": "/", + "durable": false, + "auto_delete": true, + "arguments": {} + }, + { + "name": "langohr.test.recovery.q1", + "vhost": "/", + "durable": true, + "auto_delete": true, + "arguments": {} + }, + { + "name": "langohr.tests2.queues.client-named.durable.non-exclusive.non-auto-deleted", + "vhost": "/", + "durable": true, + "auto_delete": false, + "arguments": {} + }, + { + "name": "test.tx.rollback", + "vhost": "/", + "durable": false, + "auto_delete": true, + "arguments": {} + }, + { + "name": "test-integration-declared-passive-queue", + "vhost": "/", + "durable": false, + "auto_delete": true, + "arguments": {} + }, + { + "name": "langohr.tests2.queues.client-named.non-durable.non-exclusive.auto-deleted", + "vhost": "/", + "durable": false, + "auto_delete": true, + "arguments": {} + }, + { + "name": "test.recover", + "vhost": "/", + "durable": false, + "auto_delete": true, + "arguments": {} + }, + { + "name": "amq.gen-7EZF7WjGIQFDoXexVF-e8w", + "vhost": "/", + "durable": false, + "auto_delete": true, + "arguments": { + "x-message-ttl": 1500 + } + }, + { + "name": "test.integration.channel.error", + "vhost": "/", + "durable": false, + "auto_delete": true, + "arguments": {} + }, + { + "name": "confirm", + "vhost": "/", + "durable": false, + "auto_delete": true, + "arguments": {} + }, + { + "name": "test.rabbitmq-message-ttl", + "vhost": "/", + "durable": false, + "auto_delete": true, + "arguments": { + "x-message-ttl": 100 + } + }, + { + "name": "declareWithTTL", + "vhost": "/", + "durable": false, + "auto_delete": true, + "arguments": { + "x-message-ttl": 9000000 + } + }, + { + "name": "test.tx.commit", + "vhost": "/", + "durable": false, + "auto_delete": true, + "arguments": {} + }, + { + "name": "test.get-ok", + "vhost": "/", + "durable": false, + "auto_delete": true, + "arguments": {} + }, + { + "name": "langohr.tests2.queues.non-auto-deleted1", + "vhost": "/", + "durable": false, + "auto_delete": true, + "arguments": {} + }, + { + "name": "qv3", + "vhost": "vhost3", + "durable": true, + "auto_delete": false, + "arguments": {} + } + ], + "exchanges": [ + { + "name": "bunny.tests.exchanges.fanout", + "vhost": "bunny_testbed", + "type": "fanout", + "durable": false, + "auto_delete": true, + "internal": false, + "arguments": {} + }, + { + "name": "declareArgs-dead-letter", + "vhost": "/", + "type": "fanout", + "durable": false, + "auto_delete": true, + "internal": false, + "arguments": {} + }, + { + "name": "langohr.tests.exchanges.topic5", + "vhost": "/", + "type": "topic", + "durable": false, + "auto_delete": false, + "internal": false, + "arguments": {} + }, + { + "name": "langohr.extensions.altexchanges.direct1", + "vhost": "/", + "type": "direct", + "durable": false, + "auto_delete": true, + "internal": false, + "arguments": { + "alternate-exchange": "langohr.extensions.altexchanges.fanout1" + } + }, + { + "name": "langohr.tests.exchanges.fanout1", + "vhost": "/", + "type": "fanout", + "durable": false, + "auto_delete": false, + "internal": false, + "arguments": {} + }, + { + "name": "langohr.tests.exchanges.direct3", + "vhost": "/", + "type": "direct", + "durable": false, + "auto_delete": true, + "internal": false, + "arguments": {} + }, + { + "name": "langohr.tests.exchanges.topic4", + "vhost": "/", + "type": "topic", + "durable": false, + "auto_delete": true, + "internal": false, + "arguments": {} + }, + { + "name": "langohr.test.recovery.fanout2", + "vhost": "/", + "type": "fanout", + "durable": true, + "auto_delete": false, + "internal": false, + "arguments": {} + }, + { + "name": "langohr.tests.exchanges.fanout3", + "vhost": "/", + "type": "fanout", + "durable": false, + "auto_delete": true, + "internal": false, + "arguments": {} + }, + { + "name": "langohr.tests.exchanges.direct4", + "vhost": "/", + "type": "direct", + "durable": false, + "auto_delete": false, + "internal": false, + "arguments": {} + }, + { + "name": "langohr.tests.exchanges.topic2", + "vhost": "/", + "type": "topic", + "durable": true, + "auto_delete": false, + "internal": false, + "arguments": {} + }, + { + "name": "test-integration-declared-passive-exchange", + "vhost": "/", + "type": "direct", + "durable": false, + "auto_delete": true, + "internal": false, + "arguments": {} + }, + { + "name": "test-channel-still-exists", + "vhost": "/", + "type": "direct", + "durable": false, + "auto_delete": true, + "internal": false, + "arguments": {} + }, + { + "name": "langohr.tests.exchanges.topic1", + "vhost": "/", + "type": "topic", + "durable": false, + "auto_delete": false, + "internal": false, + "arguments": {} + }, + { + "name": "langohr.tests.exchanges.fanout2", + "vhost": "/", + "type": "fanout", + "durable": true, + "auto_delete": false, + "internal": false, + "arguments": {} + }, + { + "name": "langohr.tests.exchanges.direct1", + "vhost": "/", + "type": "direct", + "durable": false, + "auto_delete": false, + "internal": false, + "arguments": {} + }, + { + "name": "langohr.tests.exchanges.direct2", + "vhost": "/", + "type": "direct", + "durable": true, + "auto_delete": false, + "internal": false, + "arguments": {} + }, + { + "name": "langohr.tests.exchanges.headers2", + "vhost": "/", + "type": "headers", + "durable": false, + "auto_delete": false, + "internal": false, + "arguments": {} + }, + { + "name": "langohr.tests.exchanges.topic3", + "vhost": "/", + "type": "topic", + "durable": false, + "auto_delete": true, + "internal": false, + "arguments": {} + }, + { + "name": "langohr.tests.exchanges.fanout4", + "vhost": "/", + "type": "fanout", + "durable": false, + "auto_delete": false, + "internal": false, + "arguments": {} + } + ], + "bindings": [ + { + "source": "amq.fanout", + "vhost": "/", + "destination": "langohr.tests2.queues.client-named.non-durable.non-exclusive.auto-deleted", + "destination_type": "queue", + "routing_key": "", + "arguments": {} + }, + { + "source": "declareArgs-dead-letter", + "vhost": "/", + "destination": "declareArgs-deliveries-dead-letter", + "destination_type": "queue", + "routing_key": "#", + "arguments": {} + } + ] +} diff --git a/deps/rabbit/test/definition_import_SUITE_data/case11.json b/deps/rabbit/test/definition_import_SUITE_data/case11.json new file mode 100644 index 0000000000..13afdf5cb5 --- /dev/null +++ b/deps/rabbit/test/definition_import_SUITE_data/case11.json @@ -0,0 +1,24 @@ +{ + "rabbit_version": "3.8.0+rc.1.5.g9148053", + "rabbitmq_version": "3.8.0+rc.1.5.g9148053", + "queues": [ + { + "name": "amq.queuebar", + "vhost": "/", + "durable": true, + "auto_delete": false, + "arguments": {} + } + ], + "exchanges": [ + { + "name": "amq.foobar", + "vhost": "/", + "type": "direct", + "durable": true, + "auto_delete": false, + "internal": false, + "arguments": {} + } + ] +} diff --git a/deps/rabbit/test/definition_import_SUITE_data/case13.json b/deps/rabbit/test/definition_import_SUITE_data/case13.json new file mode 100644 index 0000000000..726aab1e6c --- /dev/null +++ b/deps/rabbit/test/definition_import_SUITE_data/case13.json @@ -0,0 +1,55 @@ +{ + "bindings": [], + "exchanges": [], + "global_parameters": [ + { + "name": "cluster_name", + "value": "rabbit@localhost" + } + ], + "parameters": [], + "permissions": [ + { + "configure": ".*", + "read": ".*", + "user": "guest", + "vhost": "/", + "write": ".*" + } + ], + "policies": [], + "queues": [ + { + "arguments": { + "x-max-length": 991, + "x-queue-type": "quorum" + }, + "auto_delete": false, + "durable": true, + "name": "definitions.import.case13.qq.1", + "type": "quorum", + "vhost": "/" + } + ], + "rabbit_version": "3.8.6.gad0c0bd", + "rabbitmq_version": "3.8.6.gad0c0bd", + "topic_permissions": [], + "users": [ + { + "hashing_algorithm": "rabbit_password_hashing_sha256", + "name": "guest", + "password_hash": "e8lL5PHYcbv3Pd53EUoTOMnVDmsLDgVJXqSQMT+mrO4LVIdW", + "tags": "administrator" + } + ], + "vhosts": [ + { + "limits": [], + "metadata": { + "description": "Default virtual host", + "tags": [] + }, + "name": "/" + } + ] +} diff --git a/deps/rabbit/test/definition_import_SUITE_data/case2.json b/deps/rabbit/test/definition_import_SUITE_data/case2.json new file mode 100644 index 0000000000..0f0a014681 --- /dev/null +++ b/deps/rabbit/test/definition_import_SUITE_data/case2.json @@ -0,0 +1,49 @@ +{ + "rabbit_version": "3.7.0-rc.1", + "users": [ + { + "name": "guest", + "password_hash": "A0EX\/2hiwrIDKFS+nEqwbCGcVxwEkDBFF3mBfkNW53KFFk64", + "hashing_algorithm": "rabbit_password_hashing_sha256", + "tags": "administrator" + } + ], + "vhosts": [ + { + "name": "\/" + } + ], + "permissions": [ + { + "user": "guest", + "vhost": "\/", + "configure": ".*", + "write": ".*", + "read": ".*" + } + ], + "topic_permissions": [ + + ], + "parameters": [ + + ], + "global_parameters": [ + { + "name": "cluster_name", + "value": "rabbit@mercurio" + } + ], + "policies": [ + + ], + "queues": [ + + ], + "exchanges": [ + + ], + "bindings": [ + + ] +} diff --git a/deps/rabbit/test/definition_import_SUITE_data/case3.json b/deps/rabbit/test/definition_import_SUITE_data/case3.json new file mode 100644 index 0000000000..963039f254 --- /dev/null +++ b/deps/rabbit/test/definition_import_SUITE_data/case3.json @@ -0,0 +1 @@ +{"rabbit_version":"3.7.0-alpha.381","users":[{"name":"admin","password_hash":"Edl2rJd/zLC187M1SKibRoTb6+xGkvkqoKWEq0kdNUbNLyLJ","hashing_algorithm":"rabbit_password_hashing_sha256","tags":"administrator"}],"vhosts":[{"name":"/"}],"permissions":[{"user":"admin","vhost":"/","configure":".*","write":".*","read":".*"}],"topic_permissions":[],"parameters":[],"global_parameters":[{"name":"cluster_name","value":"rmq-gcp-37"}],"policies":[{"vhost":"/","name":"2-queue-replicas","pattern":".*","apply-to":"queues","definition":{"ha-mode":"exactly","ha-params":2,"ha-sync-mode":"automatic"},"priority":0}],"queues":[{"name":"wr-vary-load-lazy-persistent-1","vhost":"/","durable":true,"auto_delete":false,"arguments":{"x-queue-mode":"lazy"}},{"name":"wr-vary-load-lazy-transient-1","vhost":"/","durable":true,"auto_delete":false,"arguments":{"x-queue-mode":"lazy"}},{"name":"wr-vary-load-2","vhost":"/","durable":true,"auto_delete":false,"arguments":{"x-queue-mode":"default"}},{"name":"wr-vary-load-1","vhost":"/","durable":true,"auto_delete":false,"arguments":{"x-queue-mode":"default"}},{"name":"aliveness-test","vhost":"/","durable":false,"auto_delete":false,"arguments":{}}],"exchanges":[],"bindings":[{"source":"amq.direct","vhost":"/","destination":"wr-vary-load-2","destination_type":"queue","routing_key":"wr-vary-load-2","arguments":{}},{"source":"amq.direct","vhost":"/","destination":"wr-vary-load-lazy-persistent-1","destination_type":"queue","routing_key":"wr-vary-load-lazy-persistent-1","arguments":{}},{"source":"amq.direct","vhost":"/","destination":"wr-vary-load-lazy-transient-1","destination_type":"queue","routing_key":"wr-vary-load-lazy-transient-1","arguments":{}}]}
\ No newline at end of file diff --git a/deps/rabbit/test/definition_import_SUITE_data/case4.json b/deps/rabbit/test/definition_import_SUITE_data/case4.json new file mode 100644 index 0000000000..f5223ff3a2 --- /dev/null +++ b/deps/rabbit/test/definition_import_SUITE_data/case4.json @@ -0,0 +1,49 @@ +{ + "bindings": [], + "exchanges": [], + "parameters": [], + "permissions": [ + { + "configure": ".*", + "read": ".*", + "user": "guest", + "vhost": "/", + "write": ".*" + } + ], + "policies": [ + { + "apply-to": "all", + "definition": { + "queue-master-locator": "client-local" + }, + "name": "abc", + "pattern": "^abc\\.", + "priority": 0, + "vhost": "/" + } + ], + "queues": [ + { + "arguments": {}, + "auto_delete": false, + "durable": false, + "name": "abc.def", + "vhost": "/" + } + ], + "rabbit_version": "0.0.0", + "users": [ + { + "hashing_algorithm": "rabbit_password_hashing_sha256", + "name": "guest", + "password_hash": "QM532K822VTbYBFbwSZEnT8jkH8TT0dPsUtja6vL0myfsrmk", + "tags": "administrator" + } + ], + "vhosts": [ + { + "name": "/" + } + ] +} diff --git a/deps/rabbit/test/definition_import_SUITE_data/case5.json b/deps/rabbit/test/definition_import_SUITE_data/case5.json new file mode 100644 index 0000000000..607dfd3d1f --- /dev/null +++ b/deps/rabbit/test/definition_import_SUITE_data/case5.json @@ -0,0 +1,63 @@ +{ + "rabbit_version": "3.7.2", + "users": [ + { + "name": "guest", + "password_hash": "PD4MQV8Ivcprh1\/yUS9x7jkpbXtWIZLTQ0tvnZPncpI6Ui0a", + "hashing_algorithm": "rabbit_password_hashing_sha256", + "tags": "administrator" + } + ], + "vhosts": [ + { + "name": "\/" + }, + { + "name": "vhost2" + } + ], + "permissions": [ + { + "user": "guest", + "vhost": "\/", + "configure": ".*", + "write": ".*", + "read": ".*" + }, + { + "user": "guest", + "vhost": "vhost2", + "configure": ".*", + "write": ".*", + "read": ".*" + } + ], + "topic_permissions": [ + + ], + "parameters": [ + + ], + "global_parameters": [ + { + "name": "mqtt_port_to_vhost_mapping", + "value": { + "1883": "\/", + "1884": "vhost2" + } + }, + { + "name": "cluster_name", + "value": "rabbitmq@localhost" + } + ], + "policies": [ + ], + "queues": [ + ], + "exchanges": [ + ], + "bindings": [ + + ] +} diff --git a/deps/rabbit/test/definition_import_SUITE_data/case6.json b/deps/rabbit/test/definition_import_SUITE_data/case6.json new file mode 100644 index 0000000000..c0debb7de1 --- /dev/null +++ b/deps/rabbit/test/definition_import_SUITE_data/case6.json @@ -0,0 +1,47 @@ +{ + "rabbit_version": "3.7.3+10.g41ec73b", + "users": [ + { + "name": "guest", + "password_hash": "J+UiUxNQ3I8uPn6Lo2obWcl93VgXgbw4R+xhl3L5zHwkRFZG", + "hashing_algorithm": "rabbit_password_hashing_sha256", + "tags": "administrator" + } + ], + "vhosts": [ + { + "name": "/" + } + ], + "permissions": [ + { + "user": "guest", + "vhost": "/", + "configure": ".*", + "write": ".*", + "read": ".*" + } + ], + "topic_permissions": [], + "parameters": [ + { + "value": { + "max-queues": 456, + "max-connections": 123 + }, + "vhost": "/", + "component": "vhost-limits", + "name": "limits" + } + ], + "global_parameters": [ + { + "name": "cluster_name", + "value": "rabbit@localhost.localdomain" + } + ], + "policies": [], + "queues": [], + "exchanges": [], + "bindings": [] +}
\ No newline at end of file diff --git a/deps/rabbit/test/definition_import_SUITE_data/case7.json b/deps/rabbit/test/definition_import_SUITE_data/case7.json new file mode 100644 index 0000000000..7a8e0174ac --- /dev/null +++ b/deps/rabbit/test/definition_import_SUITE_data/case7.json @@ -0,0 +1,398 @@ + + +{ + "rabbit_version": "3.7.4", + "users": [ + { + "name": "bunny_reader", + "password_hash": "rgJkcwpypdpIVhbLDj7CaCtFVg6Dyj3yQDcCbhyn29u49c88", + "hashing_algorithm": "rabbit_password_hashing_sha256", + "tags": "" + }, + { + "name": "bunny_gem", + "password_hash": "fHFOkIlJ8iohrhN4IQXIzIDrxsOfaekv97wA1W\/0N\/uxTWjE", + "hashing_algorithm": "rabbit_password_hashing_sha256", + "tags": "" + }, + { + "name": "guest", + "password_hash": "ujflQBzsAaAfbNSLAy4y2iG9mMpgATaH5oXQfPLkxOhE1yzH", + "hashing_algorithm": "rabbit_password_hashing_sha256", + "tags": "administrator" + } + ], + "vhosts": [ + { + "name": "bunny_testbed" + }, + { + "name": "\/" + } + ], + "permissions": [ + { + "user": "bunny_reader", + "vhost": "bunny_testbed", + "configure": "^---$", + "write": "^---$", + "read": ".*" + }, + { + "user": "guest", + "vhost": "\/", + "configure": ".*", + "write": ".*", + "read": ".*" + }, + { + "user": "bunny_gem", + "vhost": "bunny_testbed", + "configure": ".*", + "write": ".*", + "read": ".*" + }, + { + "user": "guest", + "vhost": "bunny_testbed", + "configure": ".*", + "write": ".*", + "read": ".*" + } + ], + "topic_permissions": [ + + ], + "parameters": [ + { + "value": { + "pattern": "^apd\\\\.mce\\\\.estarchive.*$", + "definition": { + "max-length-bytes": 200000000 + }, + "priority": 0, + "apply-to": "queues" + }, + "vhost": "\/", + "component": "operator_policy", + "name": "apd-mce-estarchive" + } + ], + "global_parameters": [ + { + "name": "cluster_name", + "value": "rabbit@warp10" + } + ], + "policies": [ + + ], + "queues": [ + { + "name": "test", + "vhost": "\/", + "durable": true, + "auto_delete": false, + "arguments": { + + } + }, + { + "name": "reply", + "vhost": "\/", + "durable": true, + "auto_delete": false, + "arguments": { + + } + }, + { + "name": "ack-test", + "vhost": "\/", + "durable": true, + "auto_delete": false, + "arguments": { + + } + }, + { + "name": "nack-test", + "vhost": "\/", + "durable": true, + "auto_delete": false, + "arguments": { + + } + }, + { + "name": "redelivered-test", + "vhost": "\/", + "durable": true, + "auto_delete": false, + "arguments": { + + } + }, + { + "name": "unsub02", + "vhost": "\/", + "durable": true, + "auto_delete": false, + "arguments": { + + } + }, + { + "name": "known3", + "vhost": "\/", + "durable": true, + "auto_delete": false, + "arguments": { + + } + }, + { + "name": "ack-test-tx", + "vhost": "\/", + "durable": true, + "auto_delete": false, + "arguments": { + + } + }, + { + "name": "stomp-subscription-As-CQ37wutHLc9H0PmjIPw", + "vhost": "\/", + "durable": true, + "auto_delete": false, + "arguments": { + + } + }, + { + "name": "unsub01", + "vhost": "\/", + "durable": true, + "auto_delete": false, + "arguments": { + + } + }, + { + "name": "test-receipt-tx", + "vhost": "\/", + "durable": true, + "auto_delete": false, + "arguments": { + + } + }, + { + "name": "unsub03", + "vhost": "\/", + "durable": true, + "auto_delete": false, + "arguments": { + + } + }, + { + "name": "test2", + "vhost": "\/", + "durable": true, + "auto_delete": false, + "arguments": { + + } + }, + { + "name": "known", + "vhost": "\/", + "durable": true, + "auto_delete": false, + "arguments": { + + } + }, + { + "name": "unsub04", + "vhost": "\/", + "durable": true, + "auto_delete": false, + "arguments": { + + } + }, + { + "name": "stomp-subscription-j7FLeUn7ehTatYVNiBy6UA", + "vhost": "\/", + "durable": true, + "auto_delete": false, + "arguments": { + + } + }, + { + "name": "ir", + "vhost": "\/", + "durable": true, + "auto_delete": false, + "arguments": { + + } + }, + { + "name": "nack-multi", + "vhost": "\/", + "durable": true, + "auto_delete": false, + "arguments": { + + } + }, + { + "name": "test-receipt", + "vhost": "\/", + "durable": true, + "auto_delete": false, + "arguments": { + + } + }, + { + "name": "reliability", + "vhost": "\/", + "durable": true, + "auto_delete": false, + "arguments": { + + } + }, + { + "name": "duplicate-consumer-tag-test2", + "vhost": "\/", + "durable": true, + "auto_delete": false, + "arguments": { + + } + }, + { + "name": "test-multi", + "vhost": "\/", + "durable": true, + "auto_delete": false, + "arguments": { + + } + }, + { + "name": "test3", + "vhost": "\/", + "durable": true, + "auto_delete": false, + "arguments": { + + } + }, + { + "name": "stomp-subscription-tMbeqL30tjlgaXMmaFM6Ew", + "vhost": "\/", + "durable": true, + "auto_delete": false, + "arguments": { + + } + }, + { + "name": "nack-test-no-requeue", + "vhost": "\/", + "durable": true, + "auto_delete": false, + "arguments": { + + } + }, + { + "name": "ack-test-individual", + "vhost": "\/", + "durable": true, + "auto_delete": false, + "arguments": { + + } + }, + { + "name": "known2", + "vhost": "\/", + "durable": true, + "auto_delete": false, + "arguments": { + + } + }, + { + "name": "custom-header", + "vhost": "\/", + "durable": true, + "auto_delete": false, + "arguments": { + + } + }, + { + "name": "stomp-subscription-eWSXV2ty1R7VqfsnULKEkA", + "vhost": "\/", + "durable": true, + "auto_delete": false, + "arguments": { + + } + } + ], + "exchanges": [ + + ], + "bindings": [ + { + "source": "amq.topic", + "vhost": "\/", + "destination": "stomp-subscription-tMbeqL30tjlgaXMmaFM6Ew", + "destination_type": "queue", + "routing_key": "durable", + "arguments": { + + } + }, + { + "source": "amq.topic", + "vhost": "\/", + "destination": "stomp-subscription-eWSXV2ty1R7VqfsnULKEkA", + "destination_type": "queue", + "routing_key": "durable-separate", + "arguments": { + + } + }, + { + "source": "amq.topic", + "vhost": "\/", + "destination": "stomp-subscription-j7FLeUn7ehTatYVNiBy6UA", + "destination_type": "queue", + "routing_key": "durable-separate", + "arguments": { + + } + }, + { + "source": "amq.topic", + "vhost": "\/", + "destination": "stomp-subscription-As-CQ37wutHLc9H0PmjIPw", + "destination_type": "queue", + "routing_key": "durable-shared", + "arguments": { + + } + } + ] +} diff --git a/deps/rabbit/test/definition_import_SUITE_data/case8.json b/deps/rabbit/test/definition_import_SUITE_data/case8.json new file mode 100644 index 0000000000..1deb55b45c --- /dev/null +++ b/deps/rabbit/test/definition_import_SUITE_data/case8.json @@ -0,0 +1,17 @@ +{ + "rabbit_version": "3.8.0+beta.1.6.g0c7c7d9", + "parameters": [ + { + "value": { + "max-connections": 6767 + }, + "vhost": "/", + "component": "vhost-limits", + "name": "limits" + } + ], + "policies": [], + "queues": [], + "exchanges": [], + "bindings": [] +}
\ No newline at end of file diff --git a/deps/rabbit/test/definition_import_SUITE_data/case9/case9a.json b/deps/rabbit/test/definition_import_SUITE_data/case9/case9a.json new file mode 100644 index 0000000000..2e7a77962d --- /dev/null +++ b/deps/rabbit/test/definition_import_SUITE_data/case9/case9a.json @@ -0,0 +1 @@ +{"rabbit_version":"3.7.13","users":[{"name":"langohr","password_hash":"7p9PXlsYs92NlHSdNgPoDXmN77NqeGpzCTHpElq/wPS1eAEd","hashing_algorithm":"rabbit_password_hashing_sha256","tags":""},{"name":"bunny_reader","password_hash":"ExmGdjBTmQEPxcW2z+dsOuPvjFbTBiYQgMByzfpE/IIXplYG","hashing_algorithm":"rabbit_password_hashing_sha256","tags":""},{"name":"bunny_gem","password_hash":"8HH7uxmZS3FDldlYmHpFEE5+gWaeQaim8qpWIHkmNxuQK8xO","hashing_algorithm":"rabbit_password_hashing_sha256","tags":""},{"name":"guest","password_hash":"CPCbkNAHXgQ7vmrqwP9e7RWQsE8U2DqN7JA4ggS50c4LwDda","hashing_algorithm":"rabbit_password_hashing_sha256","tags":"administrator"},{"name":"temp-user","password_hash":"CfUQkDeOYDrPkACDCjoF5zySbsXPIoMgNfv7FWfEpVFGegnL","hashing_algorithm":"rabbit_password_hashing_sha256","tags":"management"}],"vhosts":[{"name":"langohr_testbed"},{"name":"bunny_testbed"},{"name":"/"}],"permissions":[{"user":"bunny_reader","vhost":"bunny_testbed","configure":"^---$","write":"^---$","read":".*"},{"user":"bunny_gem","vhost":"bunny_testbed","configure":".*","write":".*","read":".*"},{"user":"guest","vhost":"/","configure":".*","write":".*","read":".*"},{"user":"langohr","vhost":"langohr_testbed","configure":".*","write":".*","read":".*"},{"user":"guest","vhost":"bunny_testbed","configure":".*","write":".*","read":".*"},{"user":"guest","vhost":"langohr_testbed","configure":".*","write":".*","read":".*"},{"user":"langohr","vhost":"/","configure":".*","write":".*","read":".*"},{"user":"temp-user","vhost":"/","configure":".*","write":".*","read":".*"}],"topic_permissions":[],"parameters":[],"global_parameters":[{"name":"cluster_name","value":"rabbit@localhost"}],"policies":[],"queues":[{"name":"bunny.basic_consume0.1364356981103202","vhost":"bunny_testbed","durable":false,"auto_delete":true,"arguments":{}},{"name":"return","vhost":"/","durable":false,"auto_delete":true,"arguments":{}},{"name":"q1","vhost":"/","durable":true,"auto_delete":false,"arguments":{}},{"name":"declareArgs-deliveries-dead-letter","vhost":"/","durable":false,"auto_delete":true,"arguments":{}},{"name":"test.rabbitmq-basic-nack","vhost":"/","durable":false,"auto_delete":true,"arguments":{}},{"name":"langohr.test.recovery.q1","vhost":"/","durable":true,"auto_delete":true,"arguments":{}},{"name":"langohr.tests2.queues.client-named.durable.non-exclusive.non-auto-deleted","vhost":"/","durable":true,"auto_delete":false,"arguments":{}},{"name":"test.tx.rollback","vhost":"/","durable":false,"auto_delete":true,"arguments":{}},{"name":"test-integration-declared-passive-queue","vhost":"/","durable":false,"auto_delete":true,"arguments":{}},{"name":"langohr.tests2.queues.client-named.non-durable.non-exclusive.auto-deleted","vhost":"/","durable":false,"auto_delete":true,"arguments":{}},{"name":"test.recover","vhost":"/","durable":false,"auto_delete":true,"arguments":{}},{"name":"amq.gen-7EZF7WjGIQFDoXexVF-e8w","vhost":"/","durable":false,"auto_delete":true,"arguments":{"x-message-ttl":1500}},{"name":"test.integration.channel.error","vhost":"/","durable":false,"auto_delete":true,"arguments":{}},{"name":"confirm","vhost":"/","durable":false,"auto_delete":true,"arguments":{}},{"name":"test.rabbitmq-message-ttl","vhost":"/","durable":false,"auto_delete":true,"arguments":{"x-message-ttl":100}},{"name":"declareWithTTL","vhost":"/","durable":false,"auto_delete":true,"arguments":{"x-message-ttl":9000000}},{"name":"test.tx.commit","vhost":"/","durable":false,"auto_delete":true,"arguments":{}},{"name":"test.get-ok","vhost":"/","durable":false,"auto_delete":true,"arguments":{}},{"name":"langohr.tests2.queues.non-auto-deleted1","vhost":"/","durable":false,"auto_delete":true,"arguments":{}}],"exchanges":[{"name":"declareArgs-dead-letter","vhost":"/","type":"fanout","durable":false,"auto_delete":true,"internal":false,"arguments":{}},{"name":"langohr.tests.exchanges.topic5","vhost":"/","type":"topic","durable":false,"auto_delete":false,"internal":false,"arguments":{}},{"name":"langohr.extensions.altexchanges.direct1","vhost":"/","type":"direct","durable":false,"auto_delete":true,"internal":false,"arguments":{"alternate-exchange":"langohr.extensions.altexchanges.fanout1"}},{"name":"langohr.tests.exchanges.fanout1","vhost":"/","type":"fanout","durable":false,"auto_delete":false,"internal":false,"arguments":{}},{"name":"langohr.tests.exchanges.direct3","vhost":"/","type":"direct","durable":false,"auto_delete":true,"internal":false,"arguments":{}},{"name":"langohr.tests.exchanges.topic4","vhost":"/","type":"topic","durable":false,"auto_delete":true,"internal":false,"arguments":{}},{"name":"langohr.tests.exchanges.fanout3","vhost":"/","type":"fanout","durable":false,"auto_delete":true,"internal":false,"arguments":{}},{"name":"langohr.tests.exchanges.direct4","vhost":"/","type":"direct","durable":false,"auto_delete":false,"internal":false,"arguments":{}},{"name":"langohr.tests.exchanges.topic2","vhost":"/","type":"topic","durable":true,"auto_delete":false,"internal":false,"arguments":{}},{"name":"test-integration-declared-passive-exchange","vhost":"/","type":"direct","durable":false,"auto_delete":true,"internal":false,"arguments":{}},{"name":"test-channel-still-exists","vhost":"/","type":"direct","durable":false,"auto_delete":true,"internal":false,"arguments":{}},{"name":"langohr.tests.exchanges.topic1","vhost":"/","type":"topic","durable":false,"auto_delete":false,"internal":false,"arguments":{}},{"name":"langohr.tests.exchanges.fanout2","vhost":"/","type":"fanout","durable":true,"auto_delete":false,"internal":false,"arguments":{}},{"name":"langohr.tests.exchanges.direct1","vhost":"/","type":"direct","durable":false,"auto_delete":false,"internal":false,"arguments":{}},{"name":"langohr.tests.exchanges.direct2","vhost":"/","type":"direct","durable":true,"auto_delete":false,"internal":false,"arguments":{}},{"name":"langohr.tests.exchanges.headers2","vhost":"/","type":"headers","durable":false,"auto_delete":false,"internal":false,"arguments":{}},{"name":"langohr.tests.exchanges.topic3","vhost":"/","type":"topic","durable":false,"auto_delete":true,"internal":false,"arguments":{}},{"name":"langohr.test.recovery.fanout1","vhost":"/","type":"fanout","durable":true,"auto_delete":false,"internal":false,"arguments":{}},{"name":"langohr.tests.exchanges.fanout4","vhost":"/","type":"fanout","durable":false,"auto_delete":false,"internal":false,"arguments":{}}],"bindings":[{"source":"amq.fanout","vhost":"/","destination":"langohr.tests2.queues.client-named.non-durable.non-exclusive.auto-deleted","destination_type":"queue","routing_key":"","arguments":{}},{"source":"declareArgs-dead-letter","vhost":"/","destination":"declareArgs-deliveries-dead-letter","destination_type":"queue","routing_key":"#","arguments":{}}]}
\ No newline at end of file diff --git a/deps/rabbit/test/definition_import_SUITE_data/case9/case9b.json b/deps/rabbit/test/definition_import_SUITE_data/case9/case9b.json new file mode 100644 index 0000000000..7cadd58b17 --- /dev/null +++ b/deps/rabbit/test/definition_import_SUITE_data/case9/case9b.json @@ -0,0 +1 @@ +{"rabbit_version":"3.7.13","users":[{"name":"langohr","password_hash":"7p9PXlsYs92NlHSdNgPoDXmN77NqeGpzCTHpElq/wPS1eAEd","hashing_algorithm":"rabbit_password_hashing_sha256","tags":""},{"name":"bunny_reader","password_hash":"ExmGdjBTmQEPxcW2z+dsOuPvjFbTBiYQgMByzfpE/IIXplYG","hashing_algorithm":"rabbit_password_hashing_sha256","tags":""},{"name":"bunny_gem","password_hash":"8HH7uxmZS3FDldlYmHpFEE5+gWaeQaim8qpWIHkmNxuQK8xO","hashing_algorithm":"rabbit_password_hashing_sha256","tags":""},{"name":"guest2","password_hash":"E04A7cvvsaDJBezc3Sc2jCnywe9oS4DX18qFe4dwkjIr26gf","hashing_algorithm":"rabbit_password_hashing_sha256","tags":"monitoring"},{"name":"guest","password_hash":"CPCbkNAHXgQ7vmrqwP9e7RWQsE8U2DqN7JA4ggS50c4LwDda","hashing_algorithm":"rabbit_password_hashing_sha256","tags":"administrator"},{"name":"temp-user","password_hash":"CfUQkDeOYDrPkACDCjoF5zySbsXPIoMgNfv7FWfEpVFGegnL","hashing_algorithm":"rabbit_password_hashing_sha256","tags":"management"}],"vhosts":[{"name":"langohr_testbed"},{"name":"bunny_testbed"},{"name":"/"},{"name":"vhost3"}],"permissions":[{"user":"bunny_reader","vhost":"bunny_testbed","configure":"^---$","write":"^---$","read":".*"},{"user":"bunny_gem","vhost":"bunny_testbed","configure":".*","write":".*","read":".*"},{"user":"guest","vhost":"/","configure":".*","write":".*","read":".*"},{"user":"langohr","vhost":"langohr_testbed","configure":".*","write":".*","read":".*"},{"user":"guest","vhost":"bunny_testbed","configure":".*","write":".*","read":".*"},{"user":"guest","vhost":"langohr_testbed","configure":".*","write":".*","read":".*"},{"user":"guest","vhost":"vhost3","configure":".*","write":".*","read":".*"},{"user":"langohr","vhost":"/","configure":".*","write":".*","read":".*"},{"user":"temp-user","vhost":"/","configure":".*","write":".*","read":".*"}],"topic_permissions":[],"parameters":[{"value":{"max-connections":2000},"vhost":"/","component":"vhost-limits","name":"limits"}],"global_parameters":[{"name":"cluster_name","value":"rabbit@localhost"}],"policies":[],"queues":[{"name":"bunny.basic_consume0.7103611911099639","vhost":"bunny_testbed","durable":false,"auto_delete":true,"arguments":{}},{"name":"bunny.basic_consume0.6091120557781405","vhost":"bunny_testbed","durable":false,"auto_delete":true,"arguments":{}},{"name":"bunny.basic_consume0.8661861002262826","vhost":"bunny_testbed","durable":false,"auto_delete":true,"arguments":{}},{"name":"bunny.basic_consume0.3682573609392056","vhost":"bunny_testbed","durable":false,"auto_delete":true,"arguments":{}},{"name":"bunny.basic_consume0.14855593896585362","vhost":"bunny_testbed","durable":false,"auto_delete":true,"arguments":{}},{"name":"bunny.basic_consume0.9534242141484872","vhost":"bunny_testbed","durable":false,"auto_delete":true,"arguments":{}},{"name":"bunny.basic_consume0.9434723539955824","vhost":"bunny_testbed","durable":false,"auto_delete":true,"arguments":{}},{"name":"bunny.basic_consume0.12235844522013617","vhost":"bunny_testbed","durable":false,"auto_delete":true,"arguments":{}},{"name":"bunny.basic_consume0.8370997977912426","vhost":"bunny_testbed","durable":false,"auto_delete":true,"arguments":{}},{"name":"bunny.basic_consume0.4548488370639835","vhost":"bunny_testbed","durable":false,"auto_delete":true,"arguments":{}},{"name":"bunny.basic_consume0.2289868670635532","vhost":"bunny_testbed","durable":false,"auto_delete":true,"arguments":{}},{"name":"bunny.basic_consume0.00797124769641977","vhost":"bunny_testbed","durable":false,"auto_delete":true,"arguments":{}},{"name":"amq.gen-xddEPq9wHSNZKQbPK8pi3A","vhost":"bunny_testbed","durable":false,"auto_delete":false,"arguments":{}},{"name":"bunny.basic_consume0.5195700828676673","vhost":"bunny_testbed","durable":false,"auto_delete":true,"arguments":{}},{"name":"bunny.basic_consume0.3071859764599716","vhost":"bunny_testbed","durable":false,"auto_delete":true,"arguments":{}},{"name":"return","vhost":"/","durable":false,"auto_delete":true,"arguments":{}},{"name":"q1","vhost":"/","durable":true,"auto_delete":false,"arguments":{}},{"name":"declareArgs-deliveries-dead-letter","vhost":"/","durable":false,"auto_delete":true,"arguments":{}},{"name":"test.rabbitmq-basic-nack","vhost":"/","durable":false,"auto_delete":true,"arguments":{}},{"name":"langohr.test.recovery.q1","vhost":"/","durable":true,"auto_delete":true,"arguments":{}},{"name":"langohr.tests2.queues.client-named.durable.non-exclusive.non-auto-deleted","vhost":"/","durable":true,"auto_delete":false,"arguments":{}},{"name":"test.tx.rollback","vhost":"/","durable":false,"auto_delete":true,"arguments":{}},{"name":"test-integration-declared-passive-queue","vhost":"/","durable":false,"auto_delete":true,"arguments":{}},{"name":"langohr.tests2.queues.client-named.non-durable.non-exclusive.auto-deleted","vhost":"/","durable":false,"auto_delete":true,"arguments":{}},{"name":"test.recover","vhost":"/","durable":false,"auto_delete":true,"arguments":{}},{"name":"amq.gen-7EZF7WjGIQFDoXexVF-e8w","vhost":"/","durable":false,"auto_delete":true,"arguments":{"x-message-ttl":1500}},{"name":"test.integration.channel.error","vhost":"/","durable":false,"auto_delete":true,"arguments":{}},{"name":"confirm","vhost":"/","durable":false,"auto_delete":true,"arguments":{}},{"name":"test.rabbitmq-message-ttl","vhost":"/","durable":false,"auto_delete":true,"arguments":{"x-message-ttl":100}},{"name":"declareWithTTL","vhost":"/","durable":false,"auto_delete":true,"arguments":{"x-message-ttl":9000000}},{"name":"test.tx.commit","vhost":"/","durable":false,"auto_delete":true,"arguments":{}},{"name":"test.get-ok","vhost":"/","durable":false,"auto_delete":true,"arguments":{}},{"name":"langohr.tests2.queues.non-auto-deleted1","vhost":"/","durable":false,"auto_delete":true,"arguments":{}},{"name":"qv3","vhost":"vhost3","durable":true,"auto_delete":false,"arguments":{}}],"exchanges":[{"name":"bunny.tests.exchanges.fanout","vhost":"bunny_testbed","type":"fanout","durable":false,"auto_delete":true,"internal":false,"arguments":{}},{"name":"declareArgs-dead-letter","vhost":"/","type":"fanout","durable":false,"auto_delete":true,"internal":false,"arguments":{}},{"name":"langohr.tests.exchanges.topic5","vhost":"/","type":"topic","durable":false,"auto_delete":false,"internal":false,"arguments":{}},{"name":"langohr.extensions.altexchanges.direct1","vhost":"/","type":"direct","durable":false,"auto_delete":true,"internal":false,"arguments":{"alternate-exchange":"langohr.extensions.altexchanges.fanout1"}},{"name":"langohr.tests.exchanges.fanout1","vhost":"/","type":"fanout","durable":false,"auto_delete":false,"internal":false,"arguments":{}},{"name":"langohr.tests.exchanges.direct3","vhost":"/","type":"direct","durable":false,"auto_delete":true,"internal":false,"arguments":{}},{"name":"langohr.tests.exchanges.topic4","vhost":"/","type":"topic","durable":false,"auto_delete":true,"internal":false,"arguments":{}},{"name":"langohr.test.recovery.fanout2","vhost":"/","type":"fanout","durable":true,"auto_delete":false,"internal":false,"arguments":{}},{"name":"langohr.tests.exchanges.fanout3","vhost":"/","type":"fanout","durable":false,"auto_delete":true,"internal":false,"arguments":{}},{"name":"langohr.tests.exchanges.direct4","vhost":"/","type":"direct","durable":false,"auto_delete":false,"internal":false,"arguments":{}},{"name":"langohr.tests.exchanges.topic2","vhost":"/","type":"topic","durable":true,"auto_delete":false,"internal":false,"arguments":{}},{"name":"test-integration-declared-passive-exchange","vhost":"/","type":"direct","durable":false,"auto_delete":true,"internal":false,"arguments":{}},{"name":"test-channel-still-exists","vhost":"/","type":"direct","durable":false,"auto_delete":true,"internal":false,"arguments":{}},{"name":"langohr.tests.exchanges.topic1","vhost":"/","type":"topic","durable":false,"auto_delete":false,"internal":false,"arguments":{}},{"name":"langohr.tests.exchanges.fanout2","vhost":"/","type":"fanout","durable":true,"auto_delete":false,"internal":false,"arguments":{}},{"name":"langohr.tests.exchanges.direct1","vhost":"/","type":"direct","durable":false,"auto_delete":false,"internal":false,"arguments":{}},{"name":"langohr.tests.exchanges.direct2","vhost":"/","type":"direct","durable":true,"auto_delete":false,"internal":false,"arguments":{}},{"name":"langohr.tests.exchanges.headers2","vhost":"/","type":"headers","durable":false,"auto_delete":false,"internal":false,"arguments":{}},{"name":"langohr.tests.exchanges.topic3","vhost":"/","type":"topic","durable":false,"auto_delete":true,"internal":false,"arguments":{}},{"name":"langohr.tests.exchanges.fanout4","vhost":"/","type":"fanout","durable":false,"auto_delete":false,"internal":false,"arguments":{}}],"bindings":[{"source":"amq.fanout","vhost":"/","destination":"langohr.tests2.queues.client-named.non-durable.non-exclusive.auto-deleted","destination_type":"queue","routing_key":"","arguments":{}},{"source":"declareArgs-dead-letter","vhost":"/","destination":"declareArgs-deliveries-dead-letter","destination_type":"queue","routing_key":"#","arguments":{}}]}
\ No newline at end of file diff --git a/deps/rabbit/test/definition_import_SUITE_data/failing_case12.json b/deps/rabbit/test/definition_import_SUITE_data/failing_case12.json new file mode 100644 index 0000000000..6ce0366a70 --- /dev/null +++ b/deps/rabbit/test/definition_import_SUITE_data/failing_case12.json @@ -0,0 +1,24 @@ +{ + "rabbit_version": "3.8.0+rc.1.5.g9148053", + "rabbitmq_version": "3.8.0+rc.1.5.g9148053", + "queues": [ + { + "name": "amq.queuebar", + "vhost": "/", + "durable": true, + "auto_delete": false, + "arguments": {} + } + ], + "exchanges": [ + { + "name": "invalid_type", + "vhost": "/", + "type": "definitly not direct", + "durable": true, + "auto_delete": false, + "internal": false, + "arguments": {} + } + ] +} diff --git a/deps/rabbit/test/disconnect_detected_during_alarm_SUITE.erl b/deps/rabbit/test/disconnect_detected_during_alarm_SUITE.erl new file mode 100644 index 0000000000..820e13efa0 --- /dev/null +++ b/deps/rabbit/test/disconnect_detected_during_alarm_SUITE.erl @@ -0,0 +1,111 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2011-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(disconnect_detected_during_alarm_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). + +-compile(export_all). + +all() -> + [ + {group, disconnect_detected_during_alarm} + ]. + +groups() -> + [ + %% Test previously executed with the multi-node target. + {disconnect_detected_during_alarm, [], [ + disconnect_detected_during_alarm %% Trigger alarm. + ]} + ]. + +group(_) -> + []. + +%% ------------------------------------------------------------------- +%% Testsuite setup/teardown. +%% ------------------------------------------------------------------- + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(Group, Config) -> + Config1 = rabbit_ct_helpers:set_config(Config, + [ + {rmq_nodename_suffix, Group}, + {rmq_nodes_count, 1} + ]), + rabbit_ct_helpers:run_steps(Config1, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()). + +end_per_group(_Group, Config) -> + rabbit_ct_helpers:run_steps(Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()). + +end_per_group1(_, Config) -> + Config. + +init_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_started(Config, Testcase). + +end_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_finished(Config, Testcase). + +%% --------------------------------------------------------------------------- +%% Testcase +%% --------------------------------------------------------------------------- + +disconnect_detected_during_alarm(Config) -> + A = rabbit_ct_broker_helpers:get_node_config(Config, 0, nodename), + + %% Set a low memory high watermark. + rabbit_ct_broker_helpers:rabbitmqctl(Config, A, + ["set_vm_memory_high_watermark", "0.000000001"]), + + %% Open a connection and a channel. + Port = rabbit_ct_broker_helpers:get_node_config(Config, A, tcp_port_amqp), + Heartbeat = 1, + {ok, Conn} = amqp_connection:start( + #amqp_params_network{port = Port, + heartbeat = Heartbeat}), + {ok, Ch} = amqp_connection:open_channel(Conn), + + amqp_connection:register_blocked_handler(Conn, self()), + Publish = #'basic.publish'{routing_key = <<"nowhere-to-go">>}, + amqp_channel:cast(Ch, Publish, #amqp_msg{payload = <<"foobar">>}), + receive + % Check that connection was indeed blocked + #'connection.blocked'{} -> ok + after + 1000 -> exit(connection_was_not_blocked) + end, + + %% Connection is blocked, now we should forcefully kill it + {'EXIT', _} = (catch amqp_connection:close(Conn, 10)), + + ListConnections = + fun() -> + rpc:call(A, rabbit_networking, connection_info_all, []) + end, + + %% We've already disconnected, but blocked connection still should still linger on. + [SingleConn] = ListConnections(), + blocked = rabbit_misc:pget(state, SingleConn), + + %% It should definitely go away after 2 heartbeat intervals. + timer:sleep(round(2.5 * 1000 * Heartbeat)), + [] = ListConnections(), + + passed. diff --git a/deps/rabbit/test/dummy_event_receiver.erl b/deps/rabbit/test/dummy_event_receiver.erl new file mode 100644 index 0000000000..3d417b601b --- /dev/null +++ b/deps/rabbit/test/dummy_event_receiver.erl @@ -0,0 +1,49 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(dummy_event_receiver). + +-export([start/3, stop/0]). + +-export([init/1, handle_call/2, handle_event/2, handle_info/2, + terminate/2, code_change/3]). + +-include("rabbit.hrl"). + +start(Pid, Nodes, Types) -> + Oks = [ok || _ <- Nodes], + {Oks, _} = rpc:multicall(Nodes, gen_event, add_handler, + [rabbit_event, ?MODULE, [Pid, Types]]). + +stop() -> + gen_event:delete_handler(rabbit_event, ?MODULE, []). + +%%---------------------------------------------------------------------------- + +init([Pid, Types]) -> + {ok, {Pid, Types}}. + +handle_call(_Request, State) -> + {ok, not_understood, State}. + +handle_event(Event = #event{type = Type}, State = {Pid, Types}) -> + case lists:member(Type, Types) of + true -> Pid ! Event; + false -> ok + end, + {ok, State}. + +handle_info(_Info, State) -> + {ok, State}. + +terminate(_Arg, _State) -> + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%---------------------------------------------------------------------------- diff --git a/deps/rabbit/test/dummy_interceptor.erl b/deps/rabbit/test/dummy_interceptor.erl new file mode 100644 index 0000000000..6d510a3073 --- /dev/null +++ b/deps/rabbit/test/dummy_interceptor.erl @@ -0,0 +1,26 @@ +-module(dummy_interceptor). + +-behaviour(rabbit_channel_interceptor). + +-include_lib("rabbit_common/include/rabbit.hrl"). +-include_lib("rabbit_common/include/rabbit_framing.hrl"). + + +-compile(export_all). + +init(_Ch) -> + undefined. + +description() -> + [{description, + <<"Empties payload on publish">>}]. + +intercept(#'basic.publish'{} = Method, Content, _IState) -> + Content2 = Content#content{payload_fragments_rev = []}, + {Method, Content2}; + +intercept(Method, Content, _VHost) -> + {Method, Content}. + +applies_to() -> + ['basic.publish']. diff --git a/deps/rabbit/test/dummy_runtime_parameters.erl b/deps/rabbit/test/dummy_runtime_parameters.erl new file mode 100644 index 0000000000..01d0b74f95 --- /dev/null +++ b/deps/rabbit/test/dummy_runtime_parameters.erl @@ -0,0 +1,63 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(dummy_runtime_parameters). +-behaviour(rabbit_runtime_parameter). +-behaviour(rabbit_policy_validator). + +-include("rabbit.hrl"). + +-export([validate/5, notify/5, notify_clear/4]). +-export([register/0, unregister/0]). +-export([validate_policy/1]). +-export([register_policy_validator/0, unregister_policy_validator/0]). + +%---------------------------------------------------------------------------- + +register() -> + rabbit_registry:register(runtime_parameter, <<"test">>, ?MODULE). + +unregister() -> + rabbit_registry:unregister(runtime_parameter, <<"test">>). + +validate(_, <<"test">>, <<"good">>, _Term, _User) -> ok; +validate(_, <<"test">>, <<"maybe">>, <<"good">>, _User) -> ok; +validate(_, <<"test">>, <<"admin">>, _Term, none) -> ok; +validate(_, <<"test">>, <<"admin">>, _Term, User) -> + case lists:member(administrator, User#user.tags) of + true -> ok; + false -> {error, "meh", []} + end; +validate(_, <<"test">>, _, _, _) -> {error, "meh", []}. + +notify(_, _, _, _, _) -> ok. +notify_clear(_, _, _, _) -> ok. + +%---------------------------------------------------------------------------- + +register_policy_validator() -> + rabbit_registry:register(policy_validator, <<"testeven">>, ?MODULE), + rabbit_registry:register(policy_validator, <<"testpos">>, ?MODULE). + +unregister_policy_validator() -> + rabbit_registry:unregister(policy_validator, <<"testeven">>), + rabbit_registry:unregister(policy_validator, <<"testpos">>). + +validate_policy([{<<"testeven">>, Terms}]) when is_list(Terms) -> + case length(Terms) rem 2 =:= 0 of + true -> ok; + false -> {error, "meh", []} + end; + +validate_policy([{<<"testpos">>, Terms}]) when is_list(Terms) -> + case lists:all(fun (N) -> is_integer(N) andalso N > 0 end, Terms) of + true -> ok; + false -> {error, "meh", []} + end; + +validate_policy(_) -> + {error, "meh", []}. diff --git a/deps/rabbit/test/dummy_supervisor2.erl b/deps/rabbit/test/dummy_supervisor2.erl new file mode 100644 index 0000000000..354b3a0854 --- /dev/null +++ b/deps/rabbit/test/dummy_supervisor2.erl @@ -0,0 +1,32 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2011-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(dummy_supervisor2). + +-behaviour(supervisor2). + +-export([ + start_link/0, + init/1 + ]). + +start_link() -> + Pid = spawn_link(fun () -> + process_flag(trap_exit, true), + receive stop -> ok end + end), + {ok, Pid}. + +init([Timeout]) -> + {ok, {{one_for_one, 0, 1}, + [{test_sup, {supervisor2, start_link, + [{local, ?MODULE}, ?MODULE, []]}, + transient, Timeout, supervisor, [?MODULE]}]}}; +init([]) -> + {ok, {{simple_one_for_one, 0, 1}, + [{test_worker, {?MODULE, start_link, []}, + temporary, 1000, worker, [?MODULE]}]}}. diff --git a/deps/rabbit/test/dynamic_ha_SUITE.erl b/deps/rabbit/test/dynamic_ha_SUITE.erl new file mode 100644 index 0000000000..85969135b6 --- /dev/null +++ b/deps/rabbit/test/dynamic_ha_SUITE.erl @@ -0,0 +1,1034 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(dynamic_ha_SUITE). + +%% rabbit_tests:test_dynamic_mirroring() is a unit test which should +%% test the logic of what all the policies decide to do, so we don't +%% need to exhaustively test that here. What we need to test is that: +%% +%% * Going from non-mirrored to mirrored works and vice versa +%% * Changing policy can add / remove mirrors and change the master +%% * Adding a node will create a new mirror when there are not enough nodes +%% for the policy +%% * Removing a node will not create a new mirror even if the policy +%% logic wants it (since this gives us a good way to lose messages +%% on cluster shutdown, by repeated failover to new nodes) +%% +%% The first two are change_policy, the last two are change_cluster + +-include_lib("common_test/include/ct.hrl"). +-include_lib("proper/include/proper.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). +-include_lib("rabbitmq_ct_helpers/include/rabbit_assert.hrl"). + +-compile(export_all). + +-define(QNAME, <<"ha.test">>). +-define(POLICY, <<"^ha.test$">>). %% " emacs +-define(VHOST, <<"/">>). + +all() -> + [ + {group, unclustered}, + {group, clustered} + ]. + +groups() -> + [ + {unclustered, [], [ + {cluster_size_5, [], [ + change_cluster + ]} + ]}, + {clustered, [], [ + {cluster_size_2, [], [ + vhost_deletion, + force_delete_if_no_master, + promote_on_shutdown, + promote_on_failure, + follower_recovers_after_vhost_failure, + follower_recovers_after_vhost_down_and_up, + master_migrates_on_vhost_down, + follower_recovers_after_vhost_down_and_master_migrated, + queue_survive_adding_dead_vhost_mirror, + dynamic_mirroring + ]}, + {cluster_size_3, [], [ + change_policy, + rapid_change, + nodes_policy_should_pick_master_from_its_params, + promote_follower_after_standalone_restart, + queue_survive_adding_dead_vhost_mirror, + rebalance_all, + rebalance_exactly, + rebalance_nodes, + rebalance_multiple_blocked + ]} + ]} + ]. + +%% ------------------------------------------------------------------- +%% Testsuite setup/teardown. +%% ------------------------------------------------------------------- + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(unclustered, Config) -> + rabbit_ct_helpers:set_config(Config, [{rmq_nodes_clustered, false}]); +init_per_group(clustered, Config) -> + rabbit_ct_helpers:set_config(Config, [{rmq_nodes_clustered, true}]); +init_per_group(cluster_size_2, Config) -> + rabbit_ct_helpers:set_config(Config, [{rmq_nodes_count, 2}]); +init_per_group(cluster_size_3, Config) -> + rabbit_ct_helpers:set_config(Config, [{rmq_nodes_count, 3}]); +init_per_group(cluster_size_5, Config) -> + rabbit_ct_helpers:set_config(Config, [{rmq_nodes_count, 5}]). + +end_per_group(_, Config) -> + Config. + +init_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_started(Config, Testcase), + ClusterSize = ?config(rmq_nodes_count, Config), + TestNumber = rabbit_ct_helpers:testcase_number(Config, ?MODULE, Testcase), + Config1 = rabbit_ct_helpers:set_config(Config, [ + {rmq_nodename_suffix, Testcase}, + {tcp_ports_base, {skip_n_nodes, TestNumber * ClusterSize}} + ]), + rabbit_ct_helpers:run_steps(Config1, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()). + +end_per_testcase(Testcase, Config) -> + Config1 = rabbit_ct_helpers:run_steps(Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()), + rabbit_ct_helpers:testcase_finished(Config1, Testcase). + +%% ------------------------------------------------------------------- +%% Test Cases +%% ------------------------------------------------------------------- + +dynamic_mirroring(Config) -> + passed = rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, dynamic_mirroring1, [Config]). + +dynamic_mirroring1(_Config) -> + %% Just unit tests of the node selection logic, see multi node + %% tests for the rest... + Test = fun ({NewM, NewSs, ExtraSs}, Policy, Params, + {MNode, SNodes, SSNodes}, All) -> + {ok, M} = rabbit_mirror_queue_misc:module(Policy), + {NewM, NewSs0} = M:suggested_queue_nodes( + Params, MNode, SNodes, SSNodes, All), + NewSs1 = lists:sort(NewSs0), + case dm_list_match(NewSs, NewSs1, ExtraSs) of + ok -> ok; + error -> exit({no_match, NewSs, NewSs1, ExtraSs}) + end + end, + + Test({a,[b,c],0},<<"all">>,'_',{a,[], []}, [a,b,c]), + Test({a,[b,c],0},<<"all">>,'_',{a,[b,c],[b,c]},[a,b,c]), + Test({a,[b,c],0},<<"all">>,'_',{a,[d], [d]}, [a,b,c]), + + N = fun (Atoms) -> [list_to_binary(atom_to_list(A)) || A <- Atoms] end, + + %% Add a node + Test({a,[b,c],0},<<"nodes">>,N([a,b,c]),{a,[b],[b]},[a,b,c,d]), + Test({b,[a,c],0},<<"nodes">>,N([a,b,c]),{b,[a],[a]},[a,b,c,d]), + %% Add two nodes and drop one + Test({a,[b,c],0},<<"nodes">>,N([a,b,c]),{a,[d],[d]},[a,b,c,d]), + %% Don't try to include nodes that are not running + Test({a,[b], 0},<<"nodes">>,N([a,b,f]),{a,[b],[b]},[a,b,c,d]), + %% If we can't find any of the nodes listed then just keep the master + Test({a,[], 0},<<"nodes">>,N([f,g,h]),{a,[b],[b]},[a,b,c,d]), + %% And once that's happened, still keep the master even when not listed, + %% if nothing is synced + Test({a,[b,c],0},<<"nodes">>,N([b,c]), {a,[], []}, [a,b,c,d]), + Test({a,[b,c],0},<<"nodes">>,N([b,c]), {a,[b],[]}, [a,b,c,d]), + %% But if something is synced we can lose the master - but make + %% sure we pick the new master from the nodes which are synced! + Test({b,[c], 0},<<"nodes">>,N([b,c]), {a,[b],[b]},[a,b,c,d]), + Test({b,[c], 0},<<"nodes">>,N([c,b]), {a,[b],[b]},[a,b,c,d]), + + Test({a,[], 1},<<"exactly">>,2,{a,[], []}, [a,b,c,d]), + Test({a,[], 2},<<"exactly">>,3,{a,[], []}, [a,b,c,d]), + Test({a,[c], 0},<<"exactly">>,2,{a,[c], [c]}, [a,b,c,d]), + Test({a,[c], 1},<<"exactly">>,3,{a,[c], [c]}, [a,b,c,d]), + Test({a,[c], 0},<<"exactly">>,2,{a,[c,d],[c,d]},[a,b,c,d]), + Test({a,[c,d],0},<<"exactly">>,3,{a,[c,d],[c,d]},[a,b,c,d]), + + passed. + +%% Does the first list match the second where the second is required +%% to have exactly Extra superfluous items? +dm_list_match([], [], 0) -> ok; +dm_list_match(_, [], _Extra) -> error; +dm_list_match([H|T1], [H |T2], Extra) -> dm_list_match(T1, T2, Extra); +dm_list_match(L1, [_H|T2], Extra) -> dm_list_match(L1, T2, Extra - 1). + +change_policy(Config) -> + [A, B, C] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + ACh = rabbit_ct_client_helpers:open_channel(Config, A), + + %% When we first declare a queue with no policy, it's not HA. + amqp_channel:call(ACh, #'queue.declare'{queue = ?QNAME}), + timer:sleep(200), + assert_followers(A, ?QNAME, {A, ''}), + + %% Give it policy "all", it becomes HA and gets all mirrors + rabbit_ct_broker_helpers:set_ha_policy(Config, A, ?POLICY, <<"all">>), + assert_followers(A, ?QNAME, {A, [B, C]}, [{A, []}, {A, [B]}, {A, [C]}]), + + %% Give it policy "nodes", it gets specific mirrors + rabbit_ct_broker_helpers:set_ha_policy(Config, A, ?POLICY, + {<<"nodes">>, [rabbit_misc:atom_to_binary(A), + rabbit_misc:atom_to_binary(B)]}), + assert_followers(A, ?QNAME, {A, [B]}, [{A, [B, C]}]), + + %% Now explicitly change the mirrors + rabbit_ct_broker_helpers:set_ha_policy(Config, A, ?POLICY, + {<<"nodes">>, [rabbit_misc:atom_to_binary(A), + rabbit_misc:atom_to_binary(C)]}), + assert_followers(A, ?QNAME, {A, [C]}, [{A, [B, C]}]), + + %% Clear the policy, and we go back to non-mirrored + ok = rabbit_ct_broker_helpers:clear_policy(Config, A, ?POLICY), + assert_followers(A, ?QNAME, {A, ''}), + + %% Test switching "away" from an unmirrored node + rabbit_ct_broker_helpers:set_ha_policy(Config, A, ?POLICY, + {<<"nodes">>, [rabbit_misc:atom_to_binary(B), + rabbit_misc:atom_to_binary(C)]}), + assert_followers(A, ?QNAME, {B, [C]}, [{A, []}, {A, [B]}, {A, [C]}, {A, [B, C]}]), + + ok. + +change_cluster(Config) -> + [A, B, C, D, E] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + rabbit_ct_broker_helpers:cluster_nodes(Config, [A, B, C]), + ACh = rabbit_ct_client_helpers:open_channel(Config, A), + + amqp_channel:call(ACh, #'queue.declare'{queue = ?QNAME}), + assert_followers(A, ?QNAME, {A, ''}), + + %% Give it policy exactly 4, it should mirror to all 3 nodes + rabbit_ct_broker_helpers:set_ha_policy(Config, A, ?POLICY, {<<"exactly">>, 4}), + assert_followers(A, ?QNAME, {A, [B, C]}, [{A, []}, {A, [B]}, {A, [C]}]), + + %% Add D and E, D or E joins in + rabbit_ct_broker_helpers:cluster_nodes(Config, [A, D, E]), + assert_followers(A, ?QNAME, [{A, [B, C, D]}, {A, [B, C, E]}], [{A, [B, C]}]), + + %% Remove one, the other joins in + rabbit_ct_broker_helpers:stop_node(Config, D), + assert_followers(A, ?QNAME, [{A, [B, C, D]}, {A, [B, C, E]}], [{A, [B, C]}]), + + ok. + +rapid_change(Config) -> + A = rabbit_ct_broker_helpers:get_node_config(Config, 0, nodename), + ACh = rabbit_ct_client_helpers:open_channel(Config, A), + {_Pid, MRef} = spawn_monitor( + fun() -> + [rapid_amqp_ops(ACh, I) || I <- lists:seq(1, 100)] + end), + rapid_loop(Config, A, MRef), + ok. + +rapid_amqp_ops(Ch, I) -> + Payload = list_to_binary(integer_to_list(I)), + amqp_channel:call(Ch, #'queue.declare'{queue = ?QNAME}), + amqp_channel:cast(Ch, #'basic.publish'{exchange = <<"">>, + routing_key = ?QNAME}, + #amqp_msg{payload = Payload}), + amqp_channel:subscribe(Ch, #'basic.consume'{queue = ?QNAME, + no_ack = true}, self()), + receive #'basic.consume_ok'{} -> ok + end, + receive {#'basic.deliver'{}, #amqp_msg{payload = Payload}} -> + ok + end, + amqp_channel:call(Ch, #'queue.delete'{queue = ?QNAME}). + +rapid_loop(Config, Node, MRef) -> + receive + {'DOWN', MRef, process, _Pid, normal} -> + ok; + {'DOWN', MRef, process, _Pid, Reason} -> + exit({amqp_ops_died, Reason}) + after 0 -> + rabbit_ct_broker_helpers:set_ha_policy(Config, Node, ?POLICY, + <<"all">>), + ok = rabbit_ct_broker_helpers:clear_policy(Config, Node, ?POLICY), + rapid_loop(Config, Node, MRef) + end. + +queue_survive_adding_dead_vhost_mirror(Config) -> + rabbit_ct_broker_helpers:force_vhost_failure(Config, 1, <<"/">>), + NodeA = rabbit_ct_broker_helpers:get_node_config(Config, 0, nodename), + ChA = rabbit_ct_client_helpers:open_channel(Config, NodeA), + QName = <<"queue_survive_adding_dead_vhost_mirror-q-1">>, + amqp_channel:call(ChA, #'queue.declare'{queue = QName}), + Q = find_queue(QName, NodeA), + Pid = proplists:get_value(pid, Q), + rabbit_ct_broker_helpers:set_ha_policy_all(Config), + %% Queue should not fail + Q1 = find_queue(QName, NodeA), + Pid = proplists:get_value(pid, Q1). + +%% Vhost deletion needs to successfully tear down policies and queues +%% with policies. At least smoke-test that it doesn't blow up. +vhost_deletion(Config) -> + A = rabbit_ct_broker_helpers:get_node_config(Config, 0, nodename), + rabbit_ct_broker_helpers:set_ha_policy_all(Config), + ACh = rabbit_ct_client_helpers:open_channel(Config, A), + amqp_channel:call(ACh, #'queue.declare'{queue = <<"vhost_deletion-q">>}), + ok = rpc:call(A, rabbit_vhost, delete, [<<"/">>, <<"acting-user">>]), + ok. + +force_delete_if_no_master(Config) -> + [A, B] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + rabbit_ct_broker_helpers:set_ha_policy(Config, A, <<"^ha.nopromote">>, + <<"all">>), + ACh = rabbit_ct_client_helpers:open_channel(Config, A), + [begin + amqp_channel:call(ACh, #'queue.declare'{queue = Q, + durable = true}), + rabbit_ct_client_helpers:publish(ACh, Q, 10) + end || Q <- [<<"ha.nopromote.test1">>, <<"ha.nopromote.test2">>]], + ok = rabbit_ct_broker_helpers:restart_node(Config, B), + ok = rabbit_ct_broker_helpers:stop_node(Config, A), + + BCh = rabbit_ct_client_helpers:open_channel(Config, B), + ?assertExit( + {{shutdown, {server_initiated_close, 404, _}}, _}, + amqp_channel:call( + BCh, #'queue.declare'{queue = <<"ha.nopromote.test1">>, + durable = true})), + + BCh1 = rabbit_ct_client_helpers:open_channel(Config, B), + ?assertExit( + {{shutdown, {server_initiated_close, 404, _}}, _}, + amqp_channel:call( + BCh1, #'queue.declare'{queue = <<"ha.nopromote.test2">>, + durable = true})), + BCh2 = rabbit_ct_client_helpers:open_channel(Config, B), + #'queue.delete_ok'{} = + amqp_channel:call(BCh2, #'queue.delete'{queue = <<"ha.nopromote.test1">>}), + %% Delete with if_empty will fail, since we don't know if the queue is empty + ?assertExit( + {{shutdown, {server_initiated_close, 406, _}}, _}, + amqp_channel:call(BCh2, #'queue.delete'{queue = <<"ha.nopromote.test2">>, + if_empty = true})), + BCh3 = rabbit_ct_client_helpers:open_channel(Config, B), + #'queue.delete_ok'{} = + amqp_channel:call(BCh3, #'queue.delete'{queue = <<"ha.nopromote.test2">>}), + ok. + +promote_on_failure(Config) -> + [A, B] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + rabbit_ct_broker_helpers:set_ha_policy(Config, A, <<"^ha.promote">>, + <<"all">>, [{<<"ha-promote-on-failure">>, <<"always">>}]), + rabbit_ct_broker_helpers:set_ha_policy(Config, A, <<"^ha.nopromote">>, + <<"all">>, [{<<"ha-promote-on-failure">>, <<"when-synced">>}]), + + ACh = rabbit_ct_client_helpers:open_channel(Config, A), + [begin + amqp_channel:call(ACh, #'queue.declare'{queue = Q, + durable = true}), + rabbit_ct_client_helpers:publish(ACh, Q, 10) + end || Q <- [<<"ha.promote.test">>, <<"ha.nopromote.test">>]], + ok = rabbit_ct_broker_helpers:restart_node(Config, B), + ok = rabbit_ct_broker_helpers:kill_node(Config, A), + BCh = rabbit_ct_client_helpers:open_channel(Config, B), + #'queue.declare_ok'{message_count = 0} = + amqp_channel:call( + BCh, #'queue.declare'{queue = <<"ha.promote.test">>, + durable = true}), + ?assertExit( + {{shutdown, {server_initiated_close, 404, _}}, _}, + amqp_channel:call( + BCh, #'queue.declare'{queue = <<"ha.nopromote.test">>, + durable = true})), + ok = rabbit_ct_broker_helpers:start_node(Config, A), + ACh2 = rabbit_ct_client_helpers:open_channel(Config, A), + #'queue.declare_ok'{message_count = 10} = + amqp_channel:call( + ACh2, #'queue.declare'{queue = <<"ha.nopromote.test">>, + durable = true}), + ok. + +promote_on_shutdown(Config) -> + [A, B] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + rabbit_ct_broker_helpers:set_ha_policy(Config, A, <<"^ha.promote">>, + <<"all">>, [{<<"ha-promote-on-shutdown">>, <<"always">>}]), + rabbit_ct_broker_helpers:set_ha_policy(Config, A, <<"^ha.nopromote">>, + <<"all">>), + rabbit_ct_broker_helpers:set_ha_policy(Config, A, <<"^ha.nopromoteonfailure">>, + <<"all">>, [{<<"ha-promote-on-failure">>, <<"when-synced">>}, + {<<"ha-promote-on-shutdown">>, <<"always">>}]), + + ACh = rabbit_ct_client_helpers:open_channel(Config, A), + [begin + amqp_channel:call(ACh, #'queue.declare'{queue = Q, + durable = true}), + rabbit_ct_client_helpers:publish(ACh, Q, 10) + end || Q <- [<<"ha.promote.test">>, + <<"ha.nopromote.test">>, + <<"ha.nopromoteonfailure.test">>]], + ok = rabbit_ct_broker_helpers:restart_node(Config, B), + ok = rabbit_ct_broker_helpers:stop_node(Config, A), + BCh = rabbit_ct_client_helpers:open_channel(Config, B), + BCh1 = rabbit_ct_client_helpers:open_channel(Config, B), + #'queue.declare_ok'{message_count = 0} = + amqp_channel:call( + BCh, #'queue.declare'{queue = <<"ha.promote.test">>, + durable = true}), + ?assertExit( + {{shutdown, {server_initiated_close, 404, _}}, _}, + amqp_channel:call( + BCh, #'queue.declare'{queue = <<"ha.nopromote.test">>, + durable = true})), + ?assertExit( + {{shutdown, {server_initiated_close, 404, _}}, _}, + amqp_channel:call( + BCh1, #'queue.declare'{queue = <<"ha.nopromoteonfailure.test">>, + durable = true})), + ok = rabbit_ct_broker_helpers:start_node(Config, A), + ACh2 = rabbit_ct_client_helpers:open_channel(Config, A), + #'queue.declare_ok'{message_count = 10} = + amqp_channel:call( + ACh2, #'queue.declare'{queue = <<"ha.nopromote.test">>, + durable = true}), + #'queue.declare_ok'{message_count = 10} = + amqp_channel:call( + ACh2, #'queue.declare'{queue = <<"ha.nopromoteonfailure.test">>, + durable = true}), + ok. + +nodes_policy_should_pick_master_from_its_params(Config) -> + [A | _] = rabbit_ct_broker_helpers:get_node_configs(Config, + nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, A), + ?assertEqual(true, apply_policy_to_declared_queue(Config, Ch, [A], [all])), + %% --> Master: A + %% Slaves: [B, C] or [C, B] + SSPids = ?awaitMatch(SSPids when is_list(SSPids), + proplists:get_value(synchronised_slave_pids, + find_queue(?QNAME, A)), + 10000), + + %% Choose mirror that isn't the first sync mirror. Cover a bug that always + %% chose the first, even if it was not part of the policy + LastSlave = node(lists:last(SSPids)), + ?assertEqual(true, apply_policy_to_declared_queue(Config, Ch, [A], + [{nodes, [LastSlave]}])), + %% --> Master: B or C (depends on the order of current mirrors ) + %% Slaves: [] + + %% Now choose a new master that isn't synchronised. The previous + %% policy made sure that the queue only runs on one node (the last + %% from the initial synchronised list). Thus, by taking the first + %% node from this list, we know it is not synchronised. + %% + %% Because the policy doesn't cover any synchronised mirror, RabbitMQ + %% should instead use an existing synchronised mirror as the new master, + %% even though that isn't in the policy. + ?assertEqual(true, apply_policy_to_declared_queue(Config, Ch, [A], + [{nodes, [LastSlave, A]}])), + %% --> Master: B or C (same as previous policy) + %% Slaves: [A] + + NewMaster = node(erlang:hd(SSPids)), + ?assertEqual(true, apply_policy_to_declared_queue(Config, Ch, [A], + [{nodes, [NewMaster]}])), + %% --> Master: B or C (the other one compared to previous policy) + %% Slaves: [] + + amqp_channel:call(Ch, #'queue.delete'{queue = ?QNAME}), + _ = rabbit_ct_broker_helpers:clear_policy(Config, A, ?POLICY). + +follower_recovers_after_vhost_failure(Config) -> + [A, B] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + rabbit_ct_broker_helpers:set_ha_policy_all(Config), + ACh = rabbit_ct_client_helpers:open_channel(Config, A), + QName = <<"follower_recovers_after_vhost_failure-q">>, + amqp_channel:call(ACh, #'queue.declare'{queue = QName}), + timer:sleep(500), + assert_followers(A, QName, {A, [B]}, [{A, []}]), + + %% Crash vhost on a node hosting a mirror + {ok, Sup} = rabbit_ct_broker_helpers:rpc(Config, B, rabbit_vhost_sup_sup, get_vhost_sup, [<<"/">>]), + exit(Sup, foo), + + assert_followers(A, QName, {A, [B]}, [{A, []}]). + +follower_recovers_after_vhost_down_and_up(Config) -> + [A, B] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + rabbit_ct_broker_helpers:set_ha_policy_all(Config), + ACh = rabbit_ct_client_helpers:open_channel(Config, A), + QName = <<"follower_recovers_after_vhost_down_and_up-q">>, + amqp_channel:call(ACh, #'queue.declare'{queue = QName}), + timer:sleep(200), + assert_followers(A, QName, {A, [B]}, [{A, []}]), + + %% Crash vhost on a node hosting a mirror + rabbit_ct_broker_helpers:force_vhost_failure(Config, B, <<"/">>), + %% rabbit_ct_broker_helpers:force_vhost_failure/2 will retry up to 10 times to + %% make sure that the top vhost supervision tree process did go down. MK. + timer:sleep(500), + %% Vhost is back up + case rabbit_ct_broker_helpers:rpc(Config, B, rabbit_vhost_sup_sup, start_vhost, [<<"/">>]) of + {ok, _Sup} -> ok; + {error,{already_started, _Sup}} -> ok + end, + + assert_followers(A, QName, {A, [B]}, [{A, []}]). + +master_migrates_on_vhost_down(Config) -> + [A, B] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + rabbit_ct_broker_helpers:set_ha_policy_all(Config), + ACh = rabbit_ct_client_helpers:open_channel(Config, A), + QName = <<"master_migrates_on_vhost_down-q">>, + amqp_channel:call(ACh, #'queue.declare'{queue = QName}), + timer:sleep(500), + assert_followers(A, QName, {A, [B]}, [{A, []}]), + + %% Crash vhost on the node hosting queue master + rabbit_ct_broker_helpers:force_vhost_failure(Config, A, <<"/">>), + timer:sleep(500), + assert_followers(A, QName, {B, []}). + +follower_recovers_after_vhost_down_and_master_migrated(Config) -> + [A, B] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + rabbit_ct_broker_helpers:set_ha_policy_all(Config), + ACh = rabbit_ct_client_helpers:open_channel(Config, A), + QName = <<"follower_recovers_after_vhost_down_and_master_migrated-q">>, + amqp_channel:call(ACh, #'queue.declare'{queue = QName}), + timer:sleep(500), + assert_followers(A, QName, {A, [B]}, [{A, []}]), + %% Crash vhost on the node hosting queue master + rabbit_ct_broker_helpers:force_vhost_failure(Config, A, <<"/">>), + timer:sleep(500), + assert_followers(B, QName, {B, []}), + + %% Restart the vhost on the node (previously) hosting queue master + case rabbit_ct_broker_helpers:rpc(Config, A, rabbit_vhost_sup_sup, start_vhost, [<<"/">>]) of + {ok, _Sup} -> ok; + {error,{already_started, _Sup}} -> ok + end, + timer:sleep(500), + assert_followers(B, QName, {B, [A]}, [{B, []}]). + +random_policy(Config) -> + run_proper(fun prop_random_policy/1, [Config]). + +failing_random_policies(Config) -> + [A, B | _] = Nodes = rabbit_ct_broker_helpers:get_node_configs(Config, + nodename), + %% Those set of policies were found as failing by PropEr in the + %% `random_policy` test above. We add them explicitly here to make + %% sure they get tested. + ?assertEqual(true, test_random_policy(Config, Nodes, + [{nodes, [A, B]}, {nodes, [A]}])), + ?assertEqual(true, test_random_policy(Config, Nodes, + [{exactly, 3}, undefined, all, {nodes, [B]}])), + ?assertEqual(true, test_random_policy(Config, Nodes, + [all, undefined, {exactly, 2}, all, {exactly, 3}, {exactly, 3}, + undefined, {exactly, 3}, all])). + +promote_follower_after_standalone_restart(Config) -> + %% Tests that mirrors can be brought up standalone after forgetting the rest + %% of the cluster. Slave ordering should be irrelevant. + %% https://github.com/rabbitmq/rabbitmq-server/issues/1213 + [A, B, C] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + Ch = rabbit_ct_client_helpers:open_channel(Config, A), + + rabbit_ct_broker_helpers:set_ha_policy(Config, A, ?POLICY, <<"all">>), + amqp_channel:call(Ch, #'queue.declare'{queue = ?QNAME, + durable = true}), + + rabbit_ct_client_helpers:publish(Ch, ?QNAME, 15), + rabbit_ct_client_helpers:close_channel(Ch), + + rabbit_ct_helpers:await_condition(fun() -> + 15 =:= proplists:get_value(messages, find_queue(?QNAME, A)) + end, 60000), + + rabbit_ct_broker_helpers:stop_node(Config, C), + rabbit_ct_broker_helpers:stop_node(Config, B), + rabbit_ct_broker_helpers:stop_node(Config, A), + + %% Restart one mirror + forget_cluster_node(Config, B, C), + forget_cluster_node(Config, B, A), + + ok = rabbit_ct_broker_helpers:start_node(Config, B), + rabbit_ct_helpers:await_condition(fun() -> + 15 =:= proplists:get_value(messages, find_queue(?QNAME, B)) + end, 60000), + ok = rabbit_ct_broker_helpers:stop_node(Config, B), + + %% Restart the other + forget_cluster_node(Config, C, B), + forget_cluster_node(Config, C, A), + + ok = rabbit_ct_broker_helpers:start_node(Config, C), + 15 = proplists:get_value(messages, find_queue(?QNAME, C)), + ok = rabbit_ct_broker_helpers:stop_node(Config, C), + + ok. + +rebalance_all(Config) -> + [A, B, C] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + ACh = rabbit_ct_client_helpers:open_channel(Config, A), + + Q1 = <<"q1">>, + Q2 = <<"q2">>, + Q3 = <<"q3">>, + Q4 = <<"q4">>, + Q5 = <<"q5">>, + + amqp_channel:call(ACh, #'queue.declare'{queue = Q1}), + amqp_channel:call(ACh, #'queue.declare'{queue = Q2}), + amqp_channel:call(ACh, #'queue.declare'{queue = Q3}), + amqp_channel:call(ACh, #'queue.declare'{queue = Q4}), + amqp_channel:call(ACh, #'queue.declare'{queue = Q5}), + rabbit_ct_broker_helpers:set_ha_policy(Config, A, <<"q.*">>, <<"all">>), + timer:sleep(1000), + + rabbit_ct_client_helpers:publish(ACh, Q1, 5), + rabbit_ct_client_helpers:publish(ACh, Q2, 3), + assert_followers(A, Q1, {A, [B, C]}, [{A, []}, {A, [B]}, {A, [C]}]), + assert_followers(A, Q2, {A, [B, C]}, [{A, []}, {A, [B]}, {A, [C]}]), + assert_followers(A, Q3, {A, [B, C]}, [{A, []}, {A, [B]}, {A, [C]}]), + assert_followers(A, Q4, {A, [B, C]}, [{A, []}, {A, [B]}, {A, [C]}]), + assert_followers(A, Q5, {A, [B, C]}, [{A, []}, {A, [B]}, {A, [C]}]), + + {ok, Summary} = rpc:call(A, rabbit_amqqueue, rebalance, [classic, ".*", ".*"]), + + %% Check that we have at most 2 queues per node + Condition1 = fun() -> + lists:all(fun(NodeData) -> + lists:all(fun({_, V}) when is_integer(V) -> V =< 2; + (_) -> true end, + NodeData) + end, Summary) + end, + rabbit_ct_helpers:await_condition(Condition1, 60000), + + %% Check that Q1 and Q2 haven't moved + assert_followers(A, Q1, {A, [B, C]}, [{A, []}, {A, [B]}, {A, [C]}]), + assert_followers(A, Q2, {A, [B, C]}, [{A, []}, {A, [B]}, {A, [C]}]), + + ok. + +rebalance_exactly(Config) -> + [A, _, _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + ACh = rabbit_ct_client_helpers:open_channel(Config, A), + + Q1 = <<"q1">>, + Q2 = <<"q2">>, + Q3 = <<"q3">>, + Q4 = <<"q4">>, + Q5 = <<"q5">>, + + amqp_channel:call(ACh, #'queue.declare'{queue = Q1}), + amqp_channel:call(ACh, #'queue.declare'{queue = Q2}), + amqp_channel:call(ACh, #'queue.declare'{queue = Q3}), + amqp_channel:call(ACh, #'queue.declare'{queue = Q4}), + amqp_channel:call(ACh, #'queue.declare'{queue = Q5}), + rabbit_ct_broker_helpers:set_ha_policy(Config, A, <<"q.*">>, {<<"exactly">>, 2}), + timer:sleep(1000), + + %% Rebalancing happens with existing mirrors. Thus, before we + %% can verify it works as expected, we need the queues to be on + %% different mirrors. + %% + %% We only test Q3, Q4 and Q5 because the first two are expected to + %% stay where they are. + ensure_queues_are_mirrored_on_different_mirrors([Q3, Q4, Q5], A, ACh), + + rabbit_ct_client_helpers:publish(ACh, Q1, 5), + rabbit_ct_client_helpers:publish(ACh, Q2, 3), + + ?assertEqual(A, node(proplists:get_value(pid, find_queue(Q1, A)))), + ?assertEqual(A, node(proplists:get_value(pid, find_queue(Q2, A)))), + ?assertEqual(A, node(proplists:get_value(pid, find_queue(Q3, A)))), + ?assertEqual(A, node(proplists:get_value(pid, find_queue(Q4, A)))), + ?assertEqual(A, node(proplists:get_value(pid, find_queue(Q5, A)))), + + {ok, Summary} = rpc:call(A, rabbit_amqqueue, rebalance, [classic, ".*", ".*"]), + + %% Check that we have at most 2 queues per node + Condition1 = fun() -> + lists:all(fun(NodeData) -> + lists:all(fun({_, V}) when is_integer(V) -> V =< 2; + (_) -> true end, + NodeData) + end, Summary) + end, + rabbit_ct_helpers:await_condition(Condition1, 60000), + + %% Check that Q1 and Q2 haven't moved + Condition2 = fun () -> + A =:= node(proplists:get_value(pid, find_queue(Q1, A))) andalso + A =:= node(proplists:get_value(pid, find_queue(Q2, A))) + end, + rabbit_ct_helpers:await_condition(Condition2, 40000), + + ok. + +ensure_queues_are_mirrored_on_different_mirrors(Queues, Master, Ch) -> + SNodes = [node(SPid) + || Q <- Queues, + SPid <- proplists:get_value(slave_pids, find_queue(Q, Master))], + UniqueSNodes = lists:usort(SNodes), + case UniqueSNodes of + [_] -> + %% All passed queues are on the same mirror. Let's redeclare + %% one of them and test again. + Q = hd(Queues), + amqp_channel:call(Ch, #'queue.delete'{queue = Q}), + amqp_channel:call(Ch, #'queue.declare'{queue = Q}), + ensure_queues_are_mirrored_on_different_mirrors(Queues, Master, Ch); + _ -> + ok + end. + +rebalance_nodes(Config) -> + [A, B, _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + ACh = rabbit_ct_client_helpers:open_channel(Config, A), + + Q1 = <<"q1">>, + Q2 = <<"q2">>, + Q3 = <<"q3">>, + Q4 = <<"q4">>, + Q5 = <<"q5">>, + + amqp_channel:call(ACh, #'queue.declare'{queue = Q1}), + amqp_channel:call(ACh, #'queue.declare'{queue = Q2}), + amqp_channel:call(ACh, #'queue.declare'{queue = Q3}), + amqp_channel:call(ACh, #'queue.declare'{queue = Q4}), + amqp_channel:call(ACh, #'queue.declare'{queue = Q5}), + rabbit_ct_broker_helpers:set_ha_policy( + Config, A, <<"q.*">>, + {<<"nodes">>, [rabbit_misc:atom_to_binary(A), rabbit_misc:atom_to_binary(B)]}), + timer:sleep(1000), + + rabbit_ct_client_helpers:publish(ACh, Q1, 5), + rabbit_ct_client_helpers:publish(ACh, Q2, 3), + + ?assertEqual(A, node(proplists:get_value(pid, find_queue(Q1, A)))), + ?assertEqual(A, node(proplists:get_value(pid, find_queue(Q2, A)))), + ?assertEqual(A, node(proplists:get_value(pid, find_queue(Q3, A)))), + ?assertEqual(A, node(proplists:get_value(pid, find_queue(Q4, A)))), + ?assertEqual(A, node(proplists:get_value(pid, find_queue(Q5, A)))), + + {ok, Summary} = rpc:call(A, rabbit_amqqueue, rebalance, [classic, ".*", ".*"]), + + %% Check that we have at most 3 queues per node + ?assert(lists:all(fun(NodeData) -> + lists:all(fun({_, V}) when is_integer(V) -> V =< 3; + (_) -> true end, + NodeData) + end, Summary)), + %% Check that Q1 and Q2 haven't moved + ?assertEqual(A, node(proplists:get_value(pid, find_queue(Q1, A)))), + ?assertEqual(A, node(proplists:get_value(pid, find_queue(Q2, A)))), + + ok. + +rebalance_multiple_blocked(Config) -> + [A, _, _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + ACh = rabbit_ct_client_helpers:open_channel(Config, A), + Q1 = <<"q1">>, + Q2 = <<"q2">>, + Q3 = <<"q3">>, + Q4 = <<"q4">>, + Q5 = <<"q5">>, + amqp_channel:call(ACh, #'queue.declare'{queue = Q1}), + amqp_channel:call(ACh, #'queue.declare'{queue = Q2}), + amqp_channel:call(ACh, #'queue.declare'{queue = Q3}), + amqp_channel:call(ACh, #'queue.declare'{queue = Q4}), + amqp_channel:call(ACh, #'queue.declare'{queue = Q5}), + ?assertEqual(A, node(proplists:get_value(pid, find_queue(Q1, A)))), + ?assertEqual(A, node(proplists:get_value(pid, find_queue(Q2, A)))), + ?assertEqual(A, node(proplists:get_value(pid, find_queue(Q3, A)))), + ?assertEqual(A, node(proplists:get_value(pid, find_queue(Q4, A)))), + ?assertEqual(A, node(proplists:get_value(pid, find_queue(Q5, A)))), + ?assert(rabbit_ct_broker_helpers:rpc( + Config, A, + ?MODULE, rebalance_multiple_blocked1, [Config])). + +rebalance_multiple_blocked1(_) -> + Parent = self(), + Fun = fun() -> + Parent ! rabbit_amqqueue:rebalance(classic, ".*", ".*") + end, + spawn(Fun), + spawn(Fun), + Rets = [receive Ret1 -> Ret1 end, + receive Ret2 -> Ret2 end], + lists:member({error, rebalance_in_progress}, Rets). + +%%---------------------------------------------------------------------------- + +assert_followers(RPCNode, QName, Exp) -> + assert_followers(RPCNode, QName, Exp, []). + +assert_followers(RPCNode, QName, Exp, PermittedIntermediate) -> + assert_followers0(RPCNode, QName, Exp, + [{get(previous_exp_m_node), get(previous_exp_s_nodes)} | + PermittedIntermediate], 1000). + +assert_followers0(_RPCNode, _QName, [], _PermittedIntermediate, _Attempts) -> + error(invalid_expectation); +assert_followers0(RPCNode, QName, [{ExpMNode, ExpSNodes}|T], PermittedIntermediate, Attempts) -> + case assert_followers1(RPCNode, QName, {ExpMNode, ExpSNodes}, PermittedIntermediate, Attempts, nofail) of + ok -> + ok; + failed -> + assert_followers0(RPCNode, QName, T, PermittedIntermediate, Attempts - 1) + end; +assert_followers0(RPCNode, QName, {ExpMNode, ExpSNodes}, PermittedIntermediate, Attempts) -> + assert_followers1(RPCNode, QName, {ExpMNode, ExpSNodes}, PermittedIntermediate, Attempts, fail). + +assert_followers1(_RPCNode, _QName, _Exp, _PermittedIntermediate, 0, fail) -> + error(give_up_waiting_for_followers); +assert_followers1(_RPCNode, _QName, _Exp, _PermittedIntermediate, 0, nofail) -> + failed; +assert_followers1(RPCNode, QName, {ExpMNode, ExpSNodes}, PermittedIntermediate, Attempts, FastFail) -> + Q = find_queue(QName, RPCNode), + Pid = proplists:get_value(pid, Q), + SPids = proplists:get_value(slave_pids, Q), + ActMNode = node(Pid), + ActSNodes = case SPids of + '' -> ''; + _ -> [node(SPid) || SPid <- SPids] + end, + case ExpMNode =:= ActMNode andalso equal_list(ExpSNodes, ActSNodes) of + false -> + %% It's an async change, so if nothing has changed let's + %% just wait - of course this means if something does not + %% change when expected then we time out the test which is + %% a bit tedious + case [{PermMNode, PermSNodes} || {PermMNode, PermSNodes} <- PermittedIntermediate, + PermMNode =:= ActMNode, + equal_list(PermSNodes, ActSNodes)] of + [] -> + case FastFail of + fail -> + ct:fail("Expected ~p / ~p, got ~p / ~p~nat ~p~n", + [ExpMNode, ExpSNodes, ActMNode, ActSNodes, + get_stacktrace()]); + nofail -> + failed + end; + State -> + ct:pal("Waiting to leave state ~p~n Waiting for ~p~n", + [State, {ExpMNode, ExpSNodes}]), + timer:sleep(200), + assert_followers1(RPCNode, QName, {ExpMNode, ExpSNodes}, + PermittedIntermediate, + Attempts - 1, FastFail) + end; + true -> + put(previous_exp_m_node, ExpMNode), + put(previous_exp_s_nodes, ExpSNodes), + ok + end. + +equal_list('', '') -> true; +equal_list('', _Act) -> false; +equal_list(_Exp, '') -> false; +equal_list([], []) -> true; +equal_list(_Exp, []) -> false; +equal_list([], _Act) -> false; +equal_list([H|T], Act) -> case lists:member(H, Act) of + true -> equal_list(T, Act -- [H]); + false -> false + end. + +find_queue(QName, RPCNode) -> + find_queue(QName, RPCNode, 1000). + +find_queue(QName, RPCNode, 0) -> error({did_not_find_queue, QName, RPCNode}); +find_queue(QName, RPCNode, Attempts) -> + Qs = rpc:call(RPCNode, rabbit_amqqueue, info_all, [?VHOST], infinity), + case find_queue0(QName, Qs) of + did_not_find_queue -> timer:sleep(100), + find_queue(QName, RPCNode, Attempts - 1); + Q -> Q + end. + +find_queue0(QName, Qs) -> + case [Q || Q <- Qs, proplists:get_value(name, Q) =:= + rabbit_misc:r(?VHOST, queue, QName)] of + [R] -> R; + [] -> did_not_find_queue + end. + +get_stacktrace() -> + try + throw(e) + catch + _:e:Stacktrace -> + Stacktrace + end. + +%%---------------------------------------------------------------------------- +run_proper(Fun, Args) -> + ?assertEqual(true, + proper:counterexample(erlang:apply(Fun, Args), + [{numtests, 25}, + {on_output, fun(F, A) -> ct:pal(?LOW_IMPORTANCE, F, A) end}])). + +prop_random_policy(Config) -> + Nodes = rabbit_ct_broker_helpers:get_node_configs( + Config, nodename), + ?FORALL( + Policies, non_empty(list(policy_gen(Nodes))), + test_random_policy(Config, Nodes, Policies)). + +apply_policy_to_declared_queue(Config, Ch, Nodes, Policies) -> + [NodeA | _] = Nodes, + amqp_channel:call(Ch, #'queue.declare'{queue = ?QNAME}), + %% Add some load so mirrors can be busy synchronising + rabbit_ct_client_helpers:publish(Ch, ?QNAME, 100000), + %% Apply policies in parallel on all nodes + apply_in_parallel(Config, Nodes, Policies), + %% Give it some time to generate all internal notifications + timer:sleep(2000), + %% Check the result + wait_for_last_policy(?QNAME, NodeA, Policies, 30). + +test_random_policy(Config, Nodes, Policies) -> + [NodeA | _] = Nodes, + Ch = rabbit_ct_client_helpers:open_channel(Config, NodeA), + Result = apply_policy_to_declared_queue(Config, Ch, Nodes, Policies), + %% Cleanup + amqp_channel:call(Ch, #'queue.delete'{queue = ?QNAME}), + _ = rabbit_ct_broker_helpers:clear_policy(Config, NodeA, ?POLICY), + Result. + +apply_in_parallel(Config, Nodes, Policies) -> + Self = self(), + [spawn_link(fun() -> + [begin + + apply_policy(Config, N, Policy) + end || Policy <- Policies], + Self ! parallel_task_done + end) || N <- Nodes], + [receive + parallel_task_done -> + ok + end || _ <- Nodes]. + +%% Proper generators +policy_gen(Nodes) -> + %% Stop mirroring needs to be called often to trigger rabbitmq-server#803 + frequency([{3, undefined}, + {1, all}, + {1, {nodes, nodes_gen(Nodes)}}, + {1, {exactly, choose(1, 3)}} + ]). + +nodes_gen(Nodes) -> + ?LET(List, non_empty(list(oneof(Nodes))), + sets:to_list(sets:from_list(List))). + +%% Checks +wait_for_last_policy(QueueName, NodeA, TestedPolicies, Tries) -> + %% Ensure the owner/master is able to process a call request, + %% which means that all pending casts have been processed. + %% Use the information returned by owner/master to verify the + %% test result + Info = find_queue(QueueName, NodeA), + Pid = proplists:get_value(pid, Info), + Node = node(Pid), + %% Gets owner/master + case rpc:call(Node, gen_server, call, [Pid, info], 5000) of + {badrpc, _} -> + %% The queue is probably being migrated to another node. + %% Let's wait a bit longer. + timer:sleep(1000), + wait_for_last_policy(QueueName, NodeA, TestedPolicies, Tries - 1); + Result -> + FinalInfo = case Result of + {ok, I} -> I; + _ when is_list(Result) -> + Result + end, + %% The last policy is the final state + LastPolicy = lists:last(TestedPolicies), + case verify_policy(LastPolicy, FinalInfo) of + true -> + true; + false when Tries =:= 1 -> + Policies = rpc:call(Node, rabbit_policy, list, [], 5000), + ct:pal( + "Last policy not applied:~n" + " Queue node: ~s (~p)~n" + " Queue info: ~p~n" + " Configured policies: ~p~n" + " Tested policies: ~p", + [Node, Pid, FinalInfo, Policies, TestedPolicies]), + false; + false -> + timer:sleep(1000), + wait_for_last_policy(QueueName, NodeA, TestedPolicies, + Tries - 1) + end + end. + +verify_policy(undefined, Info) -> + %% If the queue is not mirrored, it returns '' + '' == proplists:get_value(slave_pids, Info); +verify_policy(all, Info) -> + 2 == length(proplists:get_value(slave_pids, Info)); +verify_policy({exactly, 1}, Info) -> + %% If the queue is mirrored, it returns a list + [] == proplists:get_value(slave_pids, Info); +verify_policy({exactly, N}, Info) -> + (N - 1) == length(proplists:get_value(slave_pids, Info)); +verify_policy({nodes, Nodes}, Info) -> + Master = node(proplists:get_value(pid, Info)), + Slaves = [node(P) || P <- proplists:get_value(slave_pids, Info)], + lists:sort(Nodes) == lists:sort([Master | Slaves]). + +%% Policies +apply_policy(Config, N, undefined) -> + _ = rabbit_ct_broker_helpers:clear_policy(Config, N, ?POLICY); +apply_policy(Config, N, all) -> + rabbit_ct_broker_helpers:set_ha_policy( + Config, N, ?POLICY, <<"all">>, + [{<<"ha-sync-mode">>, <<"automatic">>}]); +apply_policy(Config, N, {nodes, Nodes}) -> + NNodes = [rabbit_misc:atom_to_binary(Node) || Node <- Nodes], + rabbit_ct_broker_helpers:set_ha_policy( + Config, N, ?POLICY, {<<"nodes">>, NNodes}, + [{<<"ha-sync-mode">>, <<"automatic">>}]); +apply_policy(Config, N, {exactly, Exactly}) -> + rabbit_ct_broker_helpers:set_ha_policy( + Config, N, ?POLICY, {<<"exactly">>, Exactly}, + [{<<"ha-sync-mode">>, <<"automatic">>}]). + +forget_cluster_node(Config, Node, NodeToRemove) -> + rabbit_ct_broker_helpers:rabbitmqctl( + Config, Node, ["forget_cluster_node", "--offline", NodeToRemove]). diff --git a/deps/rabbit/test/dynamic_qq_SUITE.erl b/deps/rabbit/test/dynamic_qq_SUITE.erl new file mode 100644 index 0000000000..9a8f2110d6 --- /dev/null +++ b/deps/rabbit/test/dynamic_qq_SUITE.erl @@ -0,0 +1,248 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. +%% +-module(dynamic_qq_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("proper/include/proper.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). + +-import(quorum_queue_utils, [wait_for_messages_ready/3, + ra_name/1]). + +-compile(export_all). + +all() -> + [ + {group, clustered} + ]. + +groups() -> + [ + {clustered, [], [ + {cluster_size_3, [], [ + recover_follower_after_standalone_restart, + vhost_deletion, + force_delete_if_no_consensus, + takeover_on_failure, + takeover_on_shutdown, + quorum_unaffected_after_vhost_failure + ]} + ]} + ]. + +%% ------------------------------------------------------------------- +%% Testsuite setup/teardown. +%% ------------------------------------------------------------------- + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(clustered, Config) -> + rabbit_ct_helpers:set_config(Config, [{rmq_nodes_clustered, true}]); +init_per_group(cluster_size_2, Config) -> + rabbit_ct_helpers:set_config(Config, [{rmq_nodes_count, 2}]); +init_per_group(cluster_size_3, Config) -> + rabbit_ct_helpers:set_config(Config, [{rmq_nodes_count, 3}]). + +end_per_group(_, Config) -> + Config. + +init_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_started(Config, Testcase), + ClusterSize = ?config(rmq_nodes_count, Config), + TestNumber = rabbit_ct_helpers:testcase_number(Config, ?MODULE, Testcase), + Group = proplists:get_value(name, ?config(tc_group_properties, Config)), + Q = rabbit_data_coercion:to_binary(io_lib:format("~p_~p", [Group, Testcase])), + Config1 = rabbit_ct_helpers:set_config(Config, [ + {rmq_nodename_suffix, Testcase}, + {tcp_ports_base, {skip_n_nodes, TestNumber * ClusterSize}}, + {queue_name, Q}, + {queue_args, [{<<"x-queue-type">>, longstr, <<"quorum">>}]} + ]), + Config2 = rabbit_ct_helpers:run_steps( + Config1, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()), + case rabbit_ct_broker_helpers:enable_feature_flag(Config2, quorum_queue) of + ok -> + Config2; + Skip -> + end_per_testcase(Testcase, Config2), + Skip + end. + +end_per_testcase(Testcase, Config) -> + Config1 = rabbit_ct_helpers:run_steps(Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()), + rabbit_ct_helpers:testcase_finished(Config1, Testcase). + +%% ------------------------------------------------------------------- +%% Testcases. +%% ------------------------------------------------------------------- +%% Vhost deletion needs to successfully tear down queues. +vhost_deletion(Config) -> + Node = rabbit_ct_broker_helpers:get_node_config(Config, 0, nodename), + Ch = rabbit_ct_client_helpers:open_channel(Config, Node), + QName = ?config(queue_name, Config), + Args = ?config(queue_args, Config), + amqp_channel:call(Ch, #'queue.declare'{queue = QName, + arguments = Args, + durable = true + }), + ok = rpc:call(Node, rabbit_vhost, delete, [<<"/">>, <<"acting-user">>]), + ?assertMatch([], + rabbit_ct_broker_helpers:rabbitmqctl_list( + Config, 0, ["list_queues", "name"])), + ok. + +force_delete_if_no_consensus(Config) -> + [A, B, C] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + ACh = rabbit_ct_client_helpers:open_channel(Config, A), + QName = ?config(queue_name, Config), + Args = ?config(queue_args, Config), + amqp_channel:call(ACh, #'queue.declare'{queue = QName, + arguments = Args, + durable = true + }), + rabbit_ct_client_helpers:publish(ACh, QName, 10), + ok = rabbit_ct_broker_helpers:restart_node(Config, B), + ok = rabbit_ct_broker_helpers:stop_node(Config, A), + ok = rabbit_ct_broker_helpers:stop_node(Config, C), + + BCh = rabbit_ct_client_helpers:open_channel(Config, B), + ?assertMatch( + #'queue.declare_ok'{}, + amqp_channel:call( + BCh, #'queue.declare'{queue = QName, + arguments = Args, + durable = true, + passive = true})), + BCh2 = rabbit_ct_client_helpers:open_channel(Config, B), + ?assertMatch(#'queue.delete_ok'{}, + amqp_channel:call(BCh2, #'queue.delete'{queue = QName})), + ok. + +takeover_on_failure(Config) -> + takeover_on(Config, kill_node). + +takeover_on_shutdown(Config) -> + takeover_on(Config, stop_node). + +takeover_on(Config, Fun) -> + [A, B, C] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + ACh = rabbit_ct_client_helpers:open_channel(Config, A), + QName = ?config(queue_name, Config), + Args = ?config(queue_args, Config), + amqp_channel:call(ACh, #'queue.declare'{queue = QName, + arguments = Args, + durable = true + }), + rabbit_ct_client_helpers:publish(ACh, QName, 10), + ok = rabbit_ct_broker_helpers:restart_node(Config, B), + + ok = rabbit_ct_broker_helpers:Fun(Config, C), + ok = rabbit_ct_broker_helpers:Fun(Config, A), + + BCh = rabbit_ct_client_helpers:open_channel(Config, B), + #'queue.declare_ok'{} = + amqp_channel:call( + BCh, #'queue.declare'{queue = QName, + arguments = Args, + durable = true}), + ok = rabbit_ct_broker_helpers:start_node(Config, A), + ACh2 = rabbit_ct_client_helpers:open_channel(Config, A), + #'queue.declare_ok'{message_count = 10} = + amqp_channel:call( + ACh2, #'queue.declare'{queue = QName, + arguments = Args, + durable = true}), + ok. + +quorum_unaffected_after_vhost_failure(Config) -> + [A, B, _] = Servers0 = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + Servers = lists:sort(Servers0), + + ACh = rabbit_ct_client_helpers:open_channel(Config, A), + QName = ?config(queue_name, Config), + Args = ?config(queue_args, Config), + amqp_channel:call(ACh, #'queue.declare'{queue = QName, + arguments = Args, + durable = true + }), + timer:sleep(300), + + Info0 = rpc:call(A, rabbit_quorum_queue, infos, + [rabbit_misc:r(<<"/">>, queue, QName)]), + ?assertEqual(Servers, lists:sort(proplists:get_value(online, Info0, []))), + + %% Crash vhost on both nodes + {ok, SupA} = rabbit_ct_broker_helpers:rpc(Config, A, rabbit_vhost_sup_sup, get_vhost_sup, [<<"/">>]), + exit(SupA, foo), + {ok, SupB} = rabbit_ct_broker_helpers:rpc(Config, B, rabbit_vhost_sup_sup, get_vhost_sup, [<<"/">>]), + exit(SupB, foo), + + Info = rpc:call(A, rabbit_quorum_queue, infos, + [rabbit_misc:r(<<"/">>, queue, QName)]), + ?assertEqual(Servers, lists:sort(proplists:get_value(online, Info, []))). + +recover_follower_after_standalone_restart(Config) -> + case os:getenv("SECONDARY_UMBRELLA") of + false -> + %% Tests that followers can be brought up standalone after forgetting the + %% rest of the cluster. Consensus won't be reached as there is only one node in the + %% new cluster. + Servers = [A, B, C] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + Ch = rabbit_ct_client_helpers:open_channel(Config, A), + + QName = ?config(queue_name, Config), + Args = ?config(queue_args, Config), + amqp_channel:call(Ch, #'queue.declare'{queue = QName, + arguments = Args, + durable = true + }), + + rabbit_ct_client_helpers:publish(Ch, QName, 15), + rabbit_ct_client_helpers:close_channel(Ch), + + Name = ra_name(QName), + wait_for_messages_ready(Servers, Name, 15), + + rabbit_ct_broker_helpers:stop_node(Config, C), + rabbit_ct_broker_helpers:stop_node(Config, B), + rabbit_ct_broker_helpers:stop_node(Config, A), + + %% Restart one follower + forget_cluster_node(Config, B, C), + forget_cluster_node(Config, B, A), + + ok = rabbit_ct_broker_helpers:start_node(Config, B), + wait_for_messages_ready([B], Name, 15), + ok = rabbit_ct_broker_helpers:stop_node(Config, B), + + %% Restart the other + forget_cluster_node(Config, C, B), + forget_cluster_node(Config, C, A), + + ok = rabbit_ct_broker_helpers:start_node(Config, C), + wait_for_messages_ready([C], Name, 15), + ok = rabbit_ct_broker_helpers:stop_node(Config, C), + ok; + _ -> + {skip, "cannot be run in mixed mode"} + end. + +%%---------------------------------------------------------------------------- +forget_cluster_node(Config, Node, NodeToRemove) -> + rabbit_ct_broker_helpers:rabbitmqctl( + Config, Node, ["forget_cluster_node", "--offline", NodeToRemove]). diff --git a/deps/rabbit/test/eager_sync_SUITE.erl b/deps/rabbit/test/eager_sync_SUITE.erl new file mode 100644 index 0000000000..a9e2ea2107 --- /dev/null +++ b/deps/rabbit/test/eager_sync_SUITE.erl @@ -0,0 +1,271 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(eager_sync_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). + +-compile(export_all). + +-define(QNAME, <<"ha.two.test">>). +-define(QNAME_AUTO, <<"ha.auto.test">>). +-define(MESSAGE_COUNT, 200000). + +all() -> + [ + {group, non_parallel_tests} + ]. + +groups() -> + [ + {non_parallel_tests, [], [ + eager_sync, + eager_sync_cancel, + eager_sync_auto, + eager_sync_auto_on_policy_change, + eager_sync_requeue + ]} + ]. + +%% ------------------------------------------------------------------- +%% Testsuite setup/teardown. +%% ------------------------------------------------------------------- + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(_, Config) -> + Config. + +end_per_group(_, Config) -> + Config. + +init_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_started(Config, Testcase), + ClusterSize = 3, + TestNumber = rabbit_ct_helpers:testcase_number(Config, ?MODULE, Testcase), + Config1 = rabbit_ct_helpers:set_config(Config, [ + {rmq_nodes_count, ClusterSize}, + {rmq_nodes_clustered, true}, + {rmq_nodename_suffix, Testcase}, + {tcp_ports_base, {skip_n_nodes, TestNumber * ClusterSize}} + ]), + rabbit_ct_helpers:run_steps(Config1, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps() ++ [ + fun rabbit_ct_broker_helpers:set_ha_policy_two_pos/1, + fun rabbit_ct_broker_helpers:set_ha_policy_two_pos_batch_sync/1 + ]). + +end_per_testcase(Testcase, Config) -> + Config1 = rabbit_ct_helpers:run_steps(Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()), + rabbit_ct_helpers:testcase_finished(Config1, Testcase). + +%% ------------------------------------------------------------------- +%% Testcases. +%% ------------------------------------------------------------------- + +eager_sync(Config) -> + [A, B, C] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + %% Queue is on AB but not C. + ACh = rabbit_ct_client_helpers:open_channel(Config, A), + Ch = rabbit_ct_client_helpers:open_channel(Config, C), + amqp_channel:call(ACh, #'queue.declare'{queue = ?QNAME, + durable = true}), + + %% Don't sync, lose messages + rabbit_ct_client_helpers:publish(Ch, ?QNAME, ?MESSAGE_COUNT), + restart(Config, A), + restart(Config, B), + rabbit_ct_client_helpers:consume(Ch, ?QNAME, 0), + + %% Sync, keep messages + rabbit_ct_client_helpers:publish(Ch, ?QNAME, ?MESSAGE_COUNT), + restart(Config, A), + ok = sync(C, ?QNAME), + restart(Config, B), + rabbit_ct_client_helpers:consume(Ch, ?QNAME, ?MESSAGE_COUNT), + + %% Check the no-need-to-sync path + rabbit_ct_client_helpers:publish(Ch, ?QNAME, ?MESSAGE_COUNT), + ok = sync(C, ?QNAME), + rabbit_ct_client_helpers:consume(Ch, ?QNAME, ?MESSAGE_COUNT), + + %% keep unacknowledged messages + rabbit_ct_client_helpers:publish(Ch, ?QNAME, ?MESSAGE_COUNT), + rabbit_ct_client_helpers:fetch(Ch, ?QNAME, 2), + restart(Config, A), + rabbit_ct_client_helpers:fetch(Ch, ?QNAME, 3), + sync(C, ?QNAME), + restart(Config, B), + rabbit_ct_client_helpers:consume(Ch, ?QNAME, ?MESSAGE_COUNT), + + ok. + +eager_sync_cancel(Config) -> + [A, B, C] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + %% Queue is on AB but not C. + ACh = rabbit_ct_client_helpers:open_channel(Config, A), + Ch = rabbit_ct_client_helpers:open_channel(Config, C), + + set_app_sync_batch_size(A), + set_app_sync_batch_size(B), + set_app_sync_batch_size(C), + + amqp_channel:call(ACh, #'queue.declare'{queue = ?QNAME, + durable = true}), + {ok, not_syncing} = sync_cancel(C, ?QNAME), %% Idempotence + eager_sync_cancel_test2(Config, A, B, C, Ch, 100). + +eager_sync_cancel_test2(_, _, _, _, _, 0) -> + error(no_more_attempts_left); +eager_sync_cancel_test2(Config, A, B, C, Ch, Attempts) -> + %% Sync then cancel + rabbit_ct_client_helpers:publish(Ch, ?QNAME, ?MESSAGE_COUNT), + restart(Config, A), + set_app_sync_batch_size(A), + spawn_link(fun() -> ok = sync_nowait(C, ?QNAME) end), + case wait_for_syncing(C, ?QNAME, 1) of + ok -> + case sync_cancel(C, ?QNAME) of + ok -> + wait_for_running(C, ?QNAME), + restart(Config, B), + set_app_sync_batch_size(B), + rabbit_ct_client_helpers:consume(Ch, ?QNAME, 0), + + {ok, not_syncing} = sync_cancel(C, ?QNAME), %% Idempotence + ok; + {ok, not_syncing} -> + %% Damn. Syncing finished between wait_for_syncing/3 and + %% sync_cancel/2 above. Start again. + amqp_channel:call(Ch, #'queue.purge'{queue = ?QNAME}), + eager_sync_cancel_test2(Config, A, B, C, Ch, Attempts - 1) + end; + synced_already -> + %% Damn. Syncing finished before wait_for_syncing/3. Start again. + amqp_channel:call(Ch, #'queue.purge'{queue = ?QNAME}), + eager_sync_cancel_test2(Config, A, B, C, Ch, Attempts - 1) + end. + +eager_sync_auto(Config) -> + [A, B, C] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + ACh = rabbit_ct_client_helpers:open_channel(Config, A), + Ch = rabbit_ct_client_helpers:open_channel(Config, C), + amqp_channel:call(ACh, #'queue.declare'{queue = ?QNAME_AUTO, + durable = true}), + + %% Sync automatically, don't lose messages + rabbit_ct_client_helpers:publish(Ch, ?QNAME_AUTO, ?MESSAGE_COUNT), + restart(Config, A), + wait_for_sync(C, ?QNAME_AUTO), + restart(Config, B), + wait_for_sync(C, ?QNAME_AUTO), + rabbit_ct_client_helpers:consume(Ch, ?QNAME_AUTO, ?MESSAGE_COUNT), + + ok. + +eager_sync_auto_on_policy_change(Config) -> + [A, B, C] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + %% Queue is on AB but not C. + ACh = rabbit_ct_client_helpers:open_channel(Config, A), + Ch = rabbit_ct_client_helpers:open_channel(Config, C), + amqp_channel:call(ACh, #'queue.declare'{queue = ?QNAME, + durable = true}), + + %% Sync automatically once the policy is changed to tell us to. + rabbit_ct_client_helpers:publish(Ch, ?QNAME, ?MESSAGE_COUNT), + restart(Config, A), + Params = [rabbit_misc:atom_to_binary(N) || N <- [A, B]], + rabbit_ct_broker_helpers:set_ha_policy(Config, + A, <<"^ha.two.">>, {<<"nodes">>, Params}, + [{<<"ha-sync-mode">>, <<"automatic">>}]), + wait_for_sync(C, ?QNAME), + + ok. + +eager_sync_requeue(Config) -> + [A, B, C] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + %% Queue is on AB but not C. + ACh = rabbit_ct_client_helpers:open_channel(Config, A), + Ch = rabbit_ct_client_helpers:open_channel(Config, C), + amqp_channel:call(ACh, #'queue.declare'{queue = ?QNAME, + durable = true}), + + rabbit_ct_client_helpers:publish(Ch, ?QNAME, 2), + {#'basic.get_ok'{delivery_tag = TagA}, _} = + amqp_channel:call(Ch, #'basic.get'{queue = ?QNAME}), + {#'basic.get_ok'{delivery_tag = TagB}, _} = + amqp_channel:call(Ch, #'basic.get'{queue = ?QNAME}), + amqp_channel:cast(Ch, #'basic.reject'{delivery_tag = TagA, requeue = true}), + restart(Config, B), + ok = sync(C, ?QNAME), + amqp_channel:cast(Ch, #'basic.reject'{delivery_tag = TagB, requeue = true}), + rabbit_ct_client_helpers:consume(Ch, ?QNAME, 2), + + ok. + +restart(Config, Node) -> + rabbit_ct_broker_helpers:restart_broker(Config, Node). + +sync(Node, QName) -> + case sync_nowait(Node, QName) of + ok -> wait_for_sync(Node, QName), + ok; + R -> R + end. + +sync_nowait(Node, QName) -> action(Node, sync_queue, QName). +sync_cancel(Node, QName) -> action(Node, cancel_sync_queue, QName). + +wait_for_sync(Node, QName) -> + sync_detection_SUITE:wait_for_sync_status(true, Node, QName). + +action(Node, Action, QName) -> + rabbit_control_helper:command_with_output( + Action, Node, [binary_to_list(QName)], [{"-p", "/"}]). + +queue(Node, QName) -> + QNameRes = rabbit_misc:r(<<"/">>, queue, QName), + {ok, Q} = rpc:call(Node, rabbit_amqqueue, lookup, [QNameRes]), + Q. + +wait_for_syncing(Node, QName, Target) -> + case state(Node, QName) of + {{syncing, _}, _} -> ok; + {running, Target} -> synced_already; + _ -> timer:sleep(100), + wait_for_syncing(Node, QName, Target) + end. + +wait_for_running(Node, QName) -> + case state(Node, QName) of + {running, _} -> ok; + _ -> timer:sleep(100), + wait_for_running(Node, QName) + end. + +state(Node, QName) -> + [{state, State}, {synchronised_slave_pids, Pids}] = + rpc:call(Node, rabbit_amqqueue, info, + [queue(Node, QName), [state, synchronised_slave_pids]]), + {State, length(Pids)}. + +%% eager_sync_cancel_test needs a batch size that's < ?MESSAGE_COUNT +%% in order to pass, because a SyncBatchSize >= ?MESSAGE_COUNT will +%% always finish before the test is able to cancel the sync. +set_app_sync_batch_size(Node) -> + rabbit_control_helper:command( + eval, Node, + ["application:set_env(rabbit, mirroring_sync_batch_size, 1)."]). diff --git a/deps/rabbit/test/failing_dummy_interceptor.erl b/deps/rabbit/test/failing_dummy_interceptor.erl new file mode 100644 index 0000000000..62669e7f1f --- /dev/null +++ b/deps/rabbit/test/failing_dummy_interceptor.erl @@ -0,0 +1,27 @@ +-module(failing_dummy_interceptor). + +-behaviour(rabbit_channel_interceptor). + +-include_lib("rabbit_common/include/rabbit.hrl"). +-include_lib("rabbit_common/include/rabbit_framing.hrl"). + + +-compile(export_all). + +init(_Ch) -> + timer:sleep(15500), + undefined. + +description() -> + [{description, + <<"Empties payload on publish">>}]. + +intercept(#'basic.publish'{} = Method, Content, _IState) -> + Content2 = Content#content{payload_fragments_rev = []}, + {Method, Content2}; + +intercept(Method, Content, _VHost) -> + {Method, Content}. + +applies_to() -> + ['basic.publish']. diff --git a/deps/rabbit/test/feature_flags_SUITE.erl b/deps/rabbit/test/feature_flags_SUITE.erl new file mode 100644 index 0000000000..29dfcf068b --- /dev/null +++ b/deps/rabbit/test/feature_flags_SUITE.erl @@ -0,0 +1,1156 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2018-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(feature_flags_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-export([suite/0, + all/0, + groups/0, + init_per_suite/1, + end_per_suite/1, + init_per_group/2, + end_per_group/2, + init_per_testcase/2, + end_per_testcase/2, + + registry_general_usage/1, + registry_concurrent_reloads/1, + enable_feature_flag_in_a_healthy_situation/1, + enable_unsupported_feature_flag_in_a_healthy_situation/1, + enable_feature_flag_when_ff_file_is_unwritable/1, + enable_feature_flag_with_a_network_partition/1, + mark_feature_flag_as_enabled_with_a_network_partition/1, + + clustering_ok_with_ff_disabled_everywhere/1, + clustering_ok_with_ff_enabled_on_some_nodes/1, + clustering_ok_with_ff_enabled_everywhere/1, + clustering_ok_with_new_ff_disabled/1, + clustering_denied_with_new_ff_enabled/1, + clustering_ok_with_new_ff_disabled_from_plugin_on_some_nodes/1, + clustering_ok_with_new_ff_enabled_from_plugin_on_some_nodes/1, + activating_plugin_with_new_ff_disabled/1, + activating_plugin_with_new_ff_enabled/1 + ]). + +suite() -> + [{timetrap, {minutes, 15}}]. + +all() -> + [ + {group, registry}, + {group, enabling_on_single_node}, + {group, enabling_in_cluster}, + {group, clustering}, + {group, activating_plugin} + ]. + +groups() -> + [ + {registry, [], + [ + registry_general_usage, + registry_concurrent_reloads + ]}, + {enabling_on_single_node, [], + [ + enable_feature_flag_in_a_healthy_situation, + enable_unsupported_feature_flag_in_a_healthy_situation, + enable_feature_flag_when_ff_file_is_unwritable + ]}, + {enabling_in_cluster, [], + [ + enable_feature_flag_in_a_healthy_situation, + enable_unsupported_feature_flag_in_a_healthy_situation, + enable_feature_flag_when_ff_file_is_unwritable, + enable_feature_flag_with_a_network_partition, + mark_feature_flag_as_enabled_with_a_network_partition + ]}, + {clustering, [], + [ + clustering_ok_with_ff_disabled_everywhere, + clustering_ok_with_ff_enabled_on_some_nodes, + clustering_ok_with_ff_enabled_everywhere, + clustering_ok_with_new_ff_disabled, + clustering_denied_with_new_ff_enabled, + clustering_ok_with_new_ff_disabled_from_plugin_on_some_nodes, + clustering_ok_with_new_ff_enabled_from_plugin_on_some_nodes + ]}, + {activating_plugin, [], + [ + activating_plugin_with_new_ff_disabled, + activating_plugin_with_new_ff_enabled + ]} + ]. + +%% ------------------------------------------------------------------- +%% Testsuite setup/teardown. +%% ------------------------------------------------------------------- + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config, [ + fun rabbit_ct_broker_helpers:configure_dist_proxy/1 + ]). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(enabling_on_single_node, Config) -> + rabbit_ct_helpers:set_config( + Config, + [{rmq_nodes_count, 1}]); +init_per_group(enabling_in_cluster, Config) -> + rabbit_ct_helpers:set_config( + Config, + [{rmq_nodes_count, 5}]); +init_per_group(clustering, Config) -> + Config1 = rabbit_ct_helpers:set_config( + Config, + [{rmq_nodes_count, 2}, + {rmq_nodes_clustered, false}, + {start_rmq_with_plugins_disabled, true}]), + rabbit_ct_helpers:run_setup_steps(Config1, [ + fun build_my_plugin/1, + fun work_around_cli_and_rabbit_circular_dep/1 + ]); +init_per_group(activating_plugin, Config) -> + Config1 = rabbit_ct_helpers:set_config( + Config, + [{rmq_nodes_count, 2}, + {rmq_nodes_clustered, true}, + {start_rmq_with_plugins_disabled, true}]), + rabbit_ct_helpers:run_setup_steps(Config1, [ + fun build_my_plugin/1, + fun work_around_cli_and_rabbit_circular_dep/1 + ]); +init_per_group(_, Config) -> + Config. + +end_per_group(_, Config) -> + Config. + +init_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_started(Config, Testcase), + TestNumber = rabbit_ct_helpers:testcase_number(Config, ?MODULE, Testcase), + case ?config(tc_group_properties, Config) of + [{name, registry} | _] -> + application:set_env(lager, colored, true), + application:set_env( + lager, + handlers, [{lager_console_backend, [{level, debug}]}]), + application:set_env( + lager, + extra_sinks, + [{rabbit_log_lager_event, + [{handlers, [{lager_console_backend, [{level, debug}]}]}] + }, + {rabbit_log_feature_flags_lager_event, + [{handlers, [{lager_console_backend, [{level, debug}]}]}] + }]), + lager:start(), + FeatureFlagsFile = filename:join(?config(priv_dir, Config), + rabbit_misc:format( + "feature_flags-~s", + [Testcase])), + application:set_env(rabbit, feature_flags_file, FeatureFlagsFile), + rabbit_ct_helpers:set_config( + Config, {feature_flags_file, FeatureFlagsFile}); + [{name, Name} | _] + when Name =:= enabling_on_single_node orelse + Name =:= clustering orelse + Name =:= activating_plugin -> + ClusterSize = ?config(rmq_nodes_count, Config), + Config1 = rabbit_ct_helpers:set_config( + Config, + [{rmq_nodename_suffix, Testcase}, + {tcp_ports_base, {skip_n_nodes, + TestNumber * ClusterSize}} + ]), + Config2 = rabbit_ct_helpers:merge_app_env( + Config1, + {rabbit, + [{forced_feature_flags_on_init, []}, + {log, [{file, [{level, debug}]}]}]}), + Config3 = rabbit_ct_helpers:run_steps( + Config2, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()), + case Config3 of + {skip, _} -> + Config3; + _ -> + case is_feature_flag_subsystem_available(Config3) of + true -> + %% We can declare a new feature flag at + %% runtime. All of them are supported but + %% still disabled. + declare_arbitrary_feature_flag(Config3), + Config3; + false -> + end_per_testcase(Testcase, Config3), + {skip, "Feature flags subsystem unavailable"} + end + end; + [{name, enabling_in_cluster} | _] -> + ClusterSize = ?config(rmq_nodes_count, Config), + Config1 = rabbit_ct_helpers:set_config( + Config, + [{rmq_nodename_suffix, Testcase}, + {tcp_ports_base, {skip_n_nodes, + TestNumber * ClusterSize}}, + {net_ticktime, 5} + ]), + Config2 = rabbit_ct_helpers:merge_app_env( + Config1, + {rabbit, + [{forced_feature_flags_on_init, []}, + {log, [{file, [{level, debug}]}]}]}), + Config3 = rabbit_ct_helpers:run_steps( + Config2, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()), + case Config3 of + {skip, _} -> + Config3; + _ -> + case is_feature_flag_subsystem_available(Config3) of + true -> + %% We can declare a new feature flag at + %% runtime. All of them are supported but + %% still disabled. + declare_arbitrary_feature_flag(Config3), + Config3; + false -> + end_per_testcase(Testcase, Config3), + {skip, "Feature flags subsystem unavailable"} + end + end + end. + +end_per_testcase(Testcase, Config) -> + Config1 = case ?config(tc_group_properties, Config) of + [{name, registry} | _] -> + Config; + _ -> + rabbit_ct_helpers:run_steps( + Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()) + end, + rabbit_ct_helpers:testcase_finished(Config1, Testcase). + +%% ------------------------------------------------------------------- +%% Testcases. +%% ------------------------------------------------------------------- + +-define(list_ff(Which), + lists:sort(maps:keys(rabbit_ff_registry:list(Which)))). + +registry_general_usage(_Config) -> + %% At first, the registry must be uninitialized. + ?assertNot(rabbit_ff_registry:is_registry_initialized()), + + FeatureFlags = #{ff_a => + #{desc => "Feature flag A", + stability => stable}, + ff_b => + #{desc => "Feature flag B", + stability => stable}}, + rabbit_feature_flags:inject_test_feature_flags( + feature_flags_to_app_attrs(FeatureFlags)), + + %% After initialization, it must know about the feature flags + %% declared in this testsuite. They must be disabled however. + rabbit_feature_flags:initialize_registry(), + ?assert(rabbit_ff_registry:is_registry_initialized()), + ?assertMatch([ff_a, ff_b], ?list_ff(all)), + + ?assert(rabbit_ff_registry:is_supported(ff_a)), + ?assert(rabbit_ff_registry:is_supported(ff_b)), + ?assertNot(rabbit_ff_registry:is_supported(ff_c)), + ?assertNot(rabbit_ff_registry:is_supported(ff_d)), + + ?assertEqual(erlang:map_size(rabbit_ff_registry:states()), 0), + ?assertMatch([], ?list_ff(enabled)), + ?assertMatch([], ?list_ff(state_changing)), + ?assertMatch([ff_a, ff_b], ?list_ff(disabled)), + ?assertNot(rabbit_ff_registry:is_enabled(ff_a)), + ?assertNot(rabbit_ff_registry:is_enabled(ff_b)), + ?assertNot(rabbit_ff_registry:is_enabled(ff_c)), + ?assertNot(rabbit_ff_registry:is_enabled(ff_d)), + + %% We can declare a new feature flag at runtime. All of them are + %% supported but still disabled. + NewFeatureFlags = #{ff_c => + #{desc => "Feature flag C", + provided_by => ?MODULE, + stability => stable}}, + rabbit_feature_flags:initialize_registry(NewFeatureFlags), + ?assertMatch([ff_a, ff_b, ff_c], + lists:sort(maps:keys(rabbit_ff_registry:list(all)))), + + ?assert(rabbit_ff_registry:is_supported(ff_a)), + ?assert(rabbit_ff_registry:is_supported(ff_b)), + ?assert(rabbit_ff_registry:is_supported(ff_c)), + ?assertNot(rabbit_ff_registry:is_supported(ff_d)), + + ?assertEqual(erlang:map_size(rabbit_ff_registry:states()), 0), + ?assertMatch([], ?list_ff(enabled)), + ?assertMatch([], ?list_ff(state_changing)), + ?assertMatch([ff_a, ff_b, ff_c], ?list_ff(disabled)), + ?assertNot(rabbit_ff_registry:is_enabled(ff_a)), + ?assertNot(rabbit_ff_registry:is_enabled(ff_b)), + ?assertNot(rabbit_ff_registry:is_enabled(ff_c)), + ?assertNot(rabbit_ff_registry:is_enabled(ff_d)), + + %% After enabling `ff_a`, it is actually the case. Others are + %% supported but remain disabled. + rabbit_feature_flags:initialize_registry(#{}, + #{ff_a => true}, + true), + ?assertMatch([ff_a, ff_b, ff_c], + lists:sort(maps:keys(rabbit_ff_registry:list(all)))), + + ?assert(rabbit_ff_registry:is_supported(ff_a)), + ?assert(rabbit_ff_registry:is_supported(ff_b)), + ?assert(rabbit_ff_registry:is_supported(ff_c)), + ?assertNot(rabbit_ff_registry:is_supported(ff_d)), + + ?assertMatch(#{ff_a := true}, rabbit_ff_registry:states()), + ?assertMatch([ff_a], ?list_ff(enabled)), + ?assertMatch([], ?list_ff(state_changing)), + ?assertMatch([ff_b, ff_c], ?list_ff(disabled)), + ?assert(rabbit_ff_registry:is_enabled(ff_a)), + ?assertNot(rabbit_ff_registry:is_enabled(ff_b)), + ?assertNot(rabbit_ff_registry:is_enabled(ff_c)), + ?assertNot(rabbit_ff_registry:is_enabled(ff_d)), + + %% This time, we mark the state of `ff_c` as `state_changing`. We + %% expect all other feature flag states to remain unchanged. + rabbit_feature_flags:initialize_registry(#{}, + #{ff_a => false, + ff_c => state_changing}, + true), + ?assertMatch([ff_a, ff_b, ff_c], + lists:sort(maps:keys(rabbit_ff_registry:list(all)))), + + ?assert(rabbit_ff_registry:is_supported(ff_a)), + ?assert(rabbit_ff_registry:is_supported(ff_b)), + ?assert(rabbit_ff_registry:is_supported(ff_c)), + ?assertNot(rabbit_ff_registry:is_supported(ff_d)), + + ?assertMatch(#{ff_c := state_changing}, rabbit_ff_registry:states()), + ?assertMatch([], ?list_ff(enabled)), + ?assertMatch([ff_c], ?list_ff(state_changing)), + ?assertMatch([ff_a, ff_b], ?list_ff(disabled)), + ?assertNot(rabbit_ff_registry:is_enabled(ff_a)), + ?assertNot(rabbit_ff_registry:is_enabled(ff_b)), + ?assertMatch(state_changing, rabbit_ff_registry:is_enabled(ff_c)), + ?assertNot(rabbit_ff_registry:is_enabled(ff_d)), + + %% Finally, we disable `ff_c`. All of them are supported but + %% disabled. + rabbit_feature_flags:initialize_registry(#{}, + #{ff_b => false, + ff_c => false}, + true), + ?assertMatch([ff_a, ff_b, ff_c], + lists:sort(maps:keys(rabbit_ff_registry:list(all)))), + + ?assert(rabbit_ff_registry:is_supported(ff_a)), + ?assert(rabbit_ff_registry:is_supported(ff_b)), + ?assert(rabbit_ff_registry:is_supported(ff_c)), + ?assertNot(rabbit_ff_registry:is_supported(ff_d)), + + ?assertEqual(erlang:map_size(rabbit_ff_registry:states()), 0), + ?assertMatch([], ?list_ff(enabled)), + ?assertMatch([], ?list_ff(state_changing)), + ?assertMatch([ff_a, ff_b, ff_c], ?list_ff(disabled)), + ?assertNot(rabbit_ff_registry:is_enabled(ff_a)), + ?assertNot(rabbit_ff_registry:is_enabled(ff_b)), + ?assertNot(rabbit_ff_registry:is_enabled(ff_c)), + ?assertNot(rabbit_ff_registry:is_enabled(ff_d)). + +registry_concurrent_reloads(_Config) -> + case rabbit_ff_registry:is_registry_initialized() of + true -> ok; + false -> rabbit_feature_flags:initialize_registry() + end, + ?assert(rabbit_ff_registry:is_registry_initialized()), + + Parent = self(), + + MakeName = fun(I) -> + list_to_atom(rabbit_misc:format("ff_~2..0b", [I])) + end, + + ProcIs = lists:seq(1, 10), + Fun = fun(I) -> + %% Each process will declare its own feature flag to + %% make sure that each generated registry module is + %% different, and we don't loose previously declared + %% feature flags. + Name = MakeName(I), + Desc = rabbit_misc:format("Feature flag ~b", [I]), + NewFF = #{Name => + #{desc => Desc, + stability => stable}}, + rabbit_feature_flags:initialize_registry(NewFF), + unlink(Parent) + end, + + %% Prepare feature flags which the spammer process should get at + %% some point. + FeatureFlags = #{ff_a => + #{desc => "Feature flag A", + stability => stable}, + ff_b => + #{desc => "Feature flag B", + stability => stable}}, + rabbit_feature_flags:inject_test_feature_flags( + feature_flags_to_app_attrs(FeatureFlags)), + + %% Spawn a process which heavily uses the registry. + FinalFFList = lists:sort( + maps:keys(FeatureFlags) ++ + [MakeName(I) || I <- ProcIs]), + Spammer = spawn_link(fun() -> registry_spammer([], FinalFFList) end), + rabbit_log_feature_flags:info( + ?MODULE_STRING ": Started registry spammer (~p)", + [self()]), + + %% We acquire the lock from the main process to synchronize the test + %% processes we are about to spawn. + Lock = rabbit_feature_flags:registry_loading_lock(), + ThisNode = [node()], + rabbit_log_feature_flags:info( + ?MODULE_STRING ": Acquiring registry load lock"), + global:set_lock(Lock, ThisNode), + + Pids = [begin + Pid = spawn_link(fun() -> Fun(I) end), + _ = erlang:monitor(process, Pid), + Pid + end + || I <- ProcIs], + + %% We wait for one second to make sure all processes were started + %% and already sleep on the lock. Not really "make sure" because + %% we don't have a way to verify this fact, but it must be enough, + %% right? + timer:sleep(1000), + rabbit_log_feature_flags:info( + ?MODULE_STRING ": Releasing registry load lock"), + global:del_lock(Lock, ThisNode), + + rabbit_log_feature_flags:info( + ?MODULE_STRING ": Wait for test processes to finish"), + lists:foreach( + fun(Pid) -> + receive {'DOWN', _, process, Pid, normal} -> ok end + end, + Pids), + + %% We wait for one more second to make sure the spammer sees + %% all added feature flags. + timer:sleep(1000), + + unlink(Spammer), + exit(Spammer, normal). + +registry_spammer(CurrentFeatureNames, FinalFeatureNames) -> + %% Infinite loop. + case ?list_ff(all) of + CurrentFeatureNames -> + registry_spammer(CurrentFeatureNames, FinalFeatureNames); + FinalFeatureNames -> + rabbit_log_feature_flags:info( + ?MODULE_STRING ": Registry spammer: all feature flags " + "appeared"), + registry_spammer1(FinalFeatureNames); + NewFeatureNames + when length(NewFeatureNames) > length(CurrentFeatureNames) -> + registry_spammer(NewFeatureNames, FinalFeatureNames) + end. + +registry_spammer1(FeatureNames) -> + ?assertEqual(FeatureNames, ?list_ff(all)), + registry_spammer1(FeatureNames). + +enable_feature_flag_in_a_healthy_situation(Config) -> + FeatureName = ff_from_testsuite, + ClusterSize = ?config(rmq_nodes_count, Config), + Node = ClusterSize - 1, + True = lists:duplicate(ClusterSize, true), + False = lists:duplicate(ClusterSize, false), + + %% The feature flag is supported but disabled initially. + ?assertEqual( + True, + is_feature_flag_supported(Config, FeatureName)), + ?assertEqual( + False, + is_feature_flag_enabled(Config, FeatureName)), + + %% Enabling the feature flag works. + ?assertEqual( + ok, + enable_feature_flag_on(Config, Node, FeatureName)), + ?assertEqual( + True, + is_feature_flag_enabled(Config, FeatureName)), + + %% Re-enabling the feature flag also works. + ?assertEqual( + ok, + enable_feature_flag_on(Config, Node, FeatureName)), + ?assertEqual( + True, + is_feature_flag_enabled(Config, FeatureName)). + +enable_unsupported_feature_flag_in_a_healthy_situation(Config) -> + FeatureName = unsupported_feature_flag, + ClusterSize = ?config(rmq_nodes_count, Config), + Node = ClusterSize - 1, + False = lists:duplicate(ClusterSize, false), + + %% The feature flag is unsupported and thus disabled. + ?assertEqual( + False, + is_feature_flag_supported(Config, FeatureName)), + ?assertEqual( + False, + is_feature_flag_enabled(Config, FeatureName)), + + %% Enabling the feature flag works. + ?assertEqual( + {error, unsupported}, + enable_feature_flag_on(Config, Node, FeatureName)), + ?assertEqual( + False, + is_feature_flag_enabled(Config, FeatureName)). + +enable_feature_flag_when_ff_file_is_unwritable(Config) -> + QQSupported = rabbit_ct_broker_helpers:is_feature_flag_supported( + Config, quorum_queue), + case QQSupported of + true -> do_enable_feature_flag_when_ff_file_is_unwritable(Config); + false -> {skip, "Quorum queues are unsupported"} + end. + +do_enable_feature_flag_when_ff_file_is_unwritable(Config) -> + FeatureName = quorum_queue, + ClusterSize = ?config(rmq_nodes_count, Config), + Node = ClusterSize - 1, + True = lists:duplicate(ClusterSize, true), + False = lists:duplicate(ClusterSize, false), + Files = feature_flags_files(Config), + + %% The feature flag is supported but disabled initially. + ?assertEqual( + True, + is_feature_flag_supported(Config, FeatureName)), + ?assertEqual( + False, + is_feature_flag_enabled(Config, FeatureName)), + + %% Restrict permissions on the `feature_flags` files. + [?assertEqual(ok, file:change_mode(File, 8#0444)) || File <- Files], + + %% Enabling the feature flag works. + ?assertEqual( + ok, + enable_feature_flag_on(Config, Node, FeatureName)), + ?assertEqual( + True, + is_feature_flag_enabled(Config, FeatureName)), + + %% The `feature_flags` file were not updated. + ?assertEqual( + lists:duplicate(ClusterSize, {ok, [[]]}), + [file:consult(File) || File <- feature_flags_files(Config)]), + + %% Stop all nodes and restore permissions on the `feature_flags` files. + Nodes = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + [?assertEqual(ok, rabbit_ct_broker_helpers:stop_node(Config, N)) + || N <- Nodes], + [?assertEqual(ok, file:change_mode(File, 8#0644)) || File <- Files], + + %% Restart all nodes and assert the feature flag is still enabled and + %% the `feature_flags` files were correctly repaired. + [?assertEqual(ok, rabbit_ct_broker_helpers:start_node(Config, N)) + || N <- lists:reverse(Nodes)], + + ?assertEqual( + True, + is_feature_flag_enabled(Config, FeatureName)), + ?assertEqual( + lists:duplicate(ClusterSize, {ok, [[FeatureName]]}), + [file:consult(File) || File <- feature_flags_files(Config)]). + +enable_feature_flag_with_a_network_partition(Config) -> + FeatureName = ff_from_testsuite, + ClusterSize = ?config(rmq_nodes_count, Config), + [A, B, C, D, E] = rabbit_ct_broker_helpers:get_node_configs( + Config, nodename), + True = lists:duplicate(ClusterSize, true), + False = lists:duplicate(ClusterSize, false), + + %% The feature flag is supported but disabled initially. + ?assertEqual( + True, + is_feature_flag_supported(Config, FeatureName)), + ?assertEqual( + False, + is_feature_flag_enabled(Config, FeatureName)), + + %% Isolate nodes B and E from the rest of the cluster. + NodePairs = [{B, A}, + {B, C}, + {B, D}, + {E, A}, + {E, C}, + {E, D}], + block(NodePairs), + timer:sleep(1000), + + %% Enabling the feature flag should fail in the specific case of + %% `ff_from_testsuite`, if the network is broken. + ?assertEqual( + {error, unsupported}, + enable_feature_flag_on(Config, B, FeatureName)), + ?assertEqual( + False, + is_feature_flag_enabled(Config, FeatureName)), + + %% Repair the network and try again to enable the feature flag. + unblock(NodePairs), + timer:sleep(10000), + [?assertEqual(ok, rabbit_ct_broker_helpers:stop_node(Config, N)) + || N <- [A, C, D]], + [?assertEqual(ok, rabbit_ct_broker_helpers:start_node(Config, N)) + || N <- [A, C, D]], + declare_arbitrary_feature_flag(Config), + + %% Enabling the feature flag works. + ?assertEqual( + ok, + enable_feature_flag_on(Config, B, FeatureName)), + ?assertEqual( + True, + is_feature_flag_enabled(Config, FeatureName)). + +mark_feature_flag_as_enabled_with_a_network_partition(Config) -> + FeatureName = ff_from_testsuite, + ClusterSize = ?config(rmq_nodes_count, Config), + [A, B, C, D, E] = rabbit_ct_broker_helpers:get_node_configs( + Config, nodename), + True = lists:duplicate(ClusterSize, true), + False = lists:duplicate(ClusterSize, false), + + %% The feature flag is supported but disabled initially. + ?assertEqual( + True, + is_feature_flag_supported(Config, FeatureName)), + ?assertEqual( + False, + is_feature_flag_enabled(Config, FeatureName)), + + %% Isolate node B from the rest of the cluster. + NodePairs = [{B, A}, + {B, C}, + {B, D}, + {B, E}], + block(NodePairs), + timer:sleep(1000), + + %% Mark the feature flag as enabled on all nodes from node B. This + %% is expected to timeout. + RemoteNodes = [A, C, D, E], + ?assertEqual( + {failed_to_mark_feature_flag_as_enabled_on_remote_nodes, + FeatureName, + true, + RemoteNodes}, + rabbit_ct_broker_helpers:rpc( + Config, B, + rabbit_feature_flags, mark_as_enabled_remotely, + [RemoteNodes, FeatureName, true, 20000])), + + RepairFun = fun() -> + %% Wait a few seconds before we repair the network. + timer:sleep(5000), + + %% Repair the network and try again to enable + %% the feature flag. + unblock(NodePairs), + timer:sleep(1000) + end, + spawn(RepairFun), + + %% Mark the feature flag as enabled on all nodes from node B. This + %% is expected to work this time. + ct:pal(?LOW_IMPORTANCE, + "Marking the feature flag as enabled on remote nodes...", []), + ?assertEqual( + ok, + rabbit_ct_broker_helpers:rpc( + Config, B, + rabbit_feature_flags, mark_as_enabled_remotely, + [RemoteNodes, FeatureName, true, 120000])). + +%% FIXME: Finish the testcase above ^ + +clustering_ok_with_ff_disabled_everywhere(Config) -> + %% All feature flags are disabled. Clustering the two nodes should be + %% accepted because they are compatible. + + FFSubsysOk = is_feature_flag_subsystem_available(Config), + + log_feature_flags_of_all_nodes(Config), + case FFSubsysOk of + true -> + ?assertEqual([true, true], + is_feature_flag_supported(Config, ff_from_testsuite)), + ?assertEqual([false, false], + is_feature_flag_enabled(Config, ff_from_testsuite)); + false -> + ok + end, + + ?assertEqual(Config, rabbit_ct_broker_helpers:cluster_nodes(Config)), + + log_feature_flags_of_all_nodes(Config), + case FFSubsysOk of + true -> + ?assertEqual([true, true], + is_feature_flag_supported(Config, ff_from_testsuite)), + ?assertEqual([false, false], + is_feature_flag_enabled(Config, ff_from_testsuite)); + false -> + ok + end, + ok. + +clustering_ok_with_ff_enabled_on_some_nodes(Config) -> + %% The test feature flag is enabled on node 1, but not on node 2. + %% Clustering the two nodes should be accepted because they are + %% compatible. Also, the feature flag will be enabled on node 2 as a + %% consequence. + enable_feature_flag_on(Config, 0, ff_from_testsuite), + + FFSubsysOk = is_feature_flag_subsystem_available(Config), + + log_feature_flags_of_all_nodes(Config), + case FFSubsysOk of + true -> + ?assertEqual([true, true], + is_feature_flag_supported(Config, ff_from_testsuite)), + ?assertEqual([true, false], + is_feature_flag_enabled(Config, ff_from_testsuite)); + false -> + ok + end, + + ?assertEqual(Config, rabbit_ct_broker_helpers:cluster_nodes(Config)), + + log_feature_flags_of_all_nodes(Config), + case FFSubsysOk of + true -> + ?assertEqual([true, true], + is_feature_flag_enabled(Config, ff_from_testsuite)); + false -> + ok + end, + ok. + +clustering_ok_with_ff_enabled_everywhere(Config) -> + %% The test feature flags is enabled. Clustering the two nodes + %% should be accepted because they are compatible. + enable_feature_flag_everywhere(Config, ff_from_testsuite), + + FFSubsysOk = is_feature_flag_subsystem_available(Config), + + log_feature_flags_of_all_nodes(Config), + case FFSubsysOk of + true -> + ?assertEqual([true, true], + is_feature_flag_enabled(Config, ff_from_testsuite)); + false -> + ok + end, + + ?assertEqual(Config, rabbit_ct_broker_helpers:cluster_nodes(Config)), + + log_feature_flags_of_all_nodes(Config), + case FFSubsysOk of + true -> + ?assertEqual([true, true], + is_feature_flag_enabled(Config, ff_from_testsuite)); + false -> + ok + end, + ok. + +clustering_ok_with_new_ff_disabled(Config) -> + %% We declare a new (fake) feature flag on node 1. Clustering the + %% two nodes should still be accepted because that feature flag is + %% disabled. + NewFeatureFlags = #{time_travel => + #{desc => "Time travel with RabbitMQ", + provided_by => rabbit, + stability => stable}}, + rabbit_ct_broker_helpers:rpc( + Config, 0, + rabbit_feature_flags, initialize_registry, [NewFeatureFlags]), + + FFSubsysOk = is_feature_flag_subsystem_available(Config), + + log_feature_flags_of_all_nodes(Config), + case FFSubsysOk of + true -> ?assertEqual([true, false], + is_feature_flag_supported(Config, time_travel)), + ?assertEqual([false, false], + is_feature_flag_enabled(Config, time_travel)); + false -> ok + end, + + ?assertEqual(Config, rabbit_ct_broker_helpers:cluster_nodes(Config)), + + log_feature_flags_of_all_nodes(Config), + case FFSubsysOk of + true -> ?assertEqual([false, false], + is_feature_flag_supported(Config, time_travel)), + ?assertEqual([false, false], + is_feature_flag_enabled(Config, time_travel)); + false -> ok + end, + ok. + +clustering_denied_with_new_ff_enabled(Config) -> + %% We declare a new (fake) feature flag on node 1. Clustering the + %% two nodes should then be forbidden because node 2 is sure it does + %% not support it (because the application, `rabbit` is loaded and + %% it does not have it). + NewFeatureFlags = #{time_travel => + #{desc => "Time travel with RabbitMQ", + provided_by => rabbit, + stability => stable}}, + rabbit_ct_broker_helpers:rpc( + Config, 0, + rabbit_feature_flags, initialize_registry, [NewFeatureFlags]), + enable_feature_flag_on(Config, 0, time_travel), + + FFSubsysOk = is_feature_flag_subsystem_available(Config), + + log_feature_flags_of_all_nodes(Config), + case FFSubsysOk of + true -> ?assertEqual([true, false], + is_feature_flag_supported(Config, time_travel)), + ?assertEqual([true, false], + is_feature_flag_enabled(Config, time_travel)); + false -> ok + end, + + ?assertMatch({skip, _}, rabbit_ct_broker_helpers:cluster_nodes(Config)), + + log_feature_flags_of_all_nodes(Config), + case FFSubsysOk of + true -> ?assertEqual([true, false], + is_feature_flag_supported(Config, time_travel)), + ?assertEqual([true, false], + is_feature_flag_enabled(Config, time_travel)); + false -> ok + end, + ok. + +clustering_ok_with_new_ff_disabled_from_plugin_on_some_nodes(Config) -> + %% We first enable the test plugin on node 1, then we try to cluster + %% them. Even though both nodes don't share the same feature + %% flags (the test plugin exposes one), they should be considered + %% compatible and the clustering should be allowed. + rabbit_ct_broker_helpers:enable_plugin(Config, 0, "my_plugin"), + + FFSubsysOk = is_feature_flag_subsystem_available(Config), + + log_feature_flags_of_all_nodes(Config), + case FFSubsysOk of + true -> ?assertEqual([true, false], + is_feature_flag_supported(Config, plugin_ff)), + ?assertEqual([false, false], + is_feature_flag_enabled(Config, plugin_ff)); + false -> ok + end, + + ?assertEqual(Config, rabbit_ct_broker_helpers:cluster_nodes(Config)), + + log_feature_flags_of_all_nodes(Config), + case FFSubsysOk of + true -> ?assertEqual([true, true], + is_feature_flag_supported(Config, plugin_ff)), + ?assertEqual([false, false], + is_feature_flag_enabled(Config, plugin_ff)); + false -> ok + end, + ok. + +clustering_ok_with_new_ff_enabled_from_plugin_on_some_nodes(Config) -> + %% We first enable the test plugin on node 1 and enable its feature + %% flag, then we try to cluster them. Even though both nodes don't + %% share the same feature flags (the test plugin exposes one), they + %% should be considered compatible and the clustering should be + %% allowed. + rabbit_ct_broker_helpers:enable_plugin(Config, 0, "my_plugin"), + enable_feature_flag_on(Config, 0, plugin_ff), + + FFSubsysOk = is_feature_flag_subsystem_available(Config), + + log_feature_flags_of_all_nodes(Config), + case FFSubsysOk of + true -> ?assertEqual([true, false], + is_feature_flag_supported(Config, plugin_ff)), + ?assertEqual([true, false], + is_feature_flag_enabled(Config, plugin_ff)); + false -> ok + end, + + ?assertEqual(Config, rabbit_ct_broker_helpers:cluster_nodes(Config)), + + log_feature_flags_of_all_nodes(Config), + case FFSubsysOk of + true -> ?assertEqual([true, true], + is_feature_flag_supported(Config, plugin_ff)), + ?assertEqual([true, true], + is_feature_flag_enabled(Config, plugin_ff)); + false -> ok + end, + ok. + +activating_plugin_with_new_ff_disabled(Config) -> + %% Both nodes are clustered. A new plugin is enabled on node 1 + %% and this plugin has a new feature flag node 2 does know about. + %% Enabling the plugin is allowed because nodes remain compatible, + %% as the plugin is missing on one node so it can't conflict. + + FFSubsysOk = is_feature_flag_subsystem_available(Config), + + log_feature_flags_of_all_nodes(Config), + case FFSubsysOk of + true -> ?assertEqual([false, false], + is_feature_flag_supported(Config, plugin_ff)), + ?assertEqual([false, false], + is_feature_flag_enabled(Config, plugin_ff)); + false -> ok + end, + + rabbit_ct_broker_helpers:enable_plugin(Config, 0, "my_plugin"), + + log_feature_flags_of_all_nodes(Config), + case FFSubsysOk of + true -> ?assertEqual([true, true], + is_feature_flag_supported(Config, plugin_ff)), + ?assertEqual([false, false], + is_feature_flag_enabled(Config, plugin_ff)); + false -> ok + end, + ok. + +activating_plugin_with_new_ff_enabled(Config) -> + %% Both nodes are clustered. A new plugin is enabled on node 1 + %% and this plugin has a new feature flag node 2 does know about. + %% Enabling the plugin is allowed because nodes remain compatible, + %% as the plugin is missing on one node so it can't conflict. + %% Enabling the plugin's feature flag is also permitted for this + %% same reason. + + FFSubsysOk = is_feature_flag_subsystem_available(Config), + + log_feature_flags_of_all_nodes(Config), + case FFSubsysOk of + true -> ?assertEqual([false, false], + is_feature_flag_supported(Config, plugin_ff)), + ?assertEqual([false, false], + is_feature_flag_enabled(Config, plugin_ff)); + false -> ok + end, + + rabbit_ct_broker_helpers:enable_plugin(Config, 0, "my_plugin"), + enable_feature_flag_on(Config, 0, plugin_ff), + + log_feature_flags_of_all_nodes(Config), + case FFSubsysOk of + true -> ?assertEqual([true, true], + is_feature_flag_supported(Config, plugin_ff)), + ?assertEqual([true, true], + is_feature_flag_enabled(Config, plugin_ff)); + false -> ok + end, + ok. + +%% ------------------------------------------------------------------- +%% Internal helpers. +%% ------------------------------------------------------------------- + +build_my_plugin(Config) -> + PluginSrcDir = filename:join(?config(data_dir, Config), "my_plugin"), + PluginsDir = filename:join(PluginSrcDir, "plugins"), + Config1 = rabbit_ct_helpers:set_config(Config, + [{rmq_plugins_dir, PluginsDir}]), + {MyPlugin, OtherPlugins} = list_my_plugin_plugins(PluginSrcDir), + case MyPlugin of + [] -> + DepsDir = ?config(erlang_mk_depsdir, Config), + Args = ["test-dist", + {"DEPS_DIR=~s", [DepsDir]}, + %% We clear ALL_DEPS_DIRS to make sure they are + %% not recompiled when the plugin is built. `rabbit` + %% was previously compiled with -DTEST and if it is + %% recompiled because of this plugin, it will be + %% recompiled without -DTEST: the testsuite depends + %% on test code so we can't allow that. + %% + %% Note that we do not clear the DEPS variable: + %% we need it to be correct because it is used to + %% generate `my_plugin.app` (and a RabbitMQ plugin + %% must depend on `rabbit`). + "ALL_DEPS_DIRS="], + case rabbit_ct_helpers:make(Config1, PluginSrcDir, Args) of + {ok, _} -> + {_, OtherPlugins1} = list_my_plugin_plugins(PluginSrcDir), + remove_other_plugins(PluginSrcDir, OtherPlugins1), + update_cli_path(Config1, PluginSrcDir); + {error, _} -> + {skip, "Failed to compile the `my_plugin` test plugin"} + end; + _ -> + remove_other_plugins(PluginSrcDir, OtherPlugins), + update_cli_path(Config1, PluginSrcDir) + end. + +update_cli_path(Config, PluginSrcDir) -> + SbinDir = filename:join(PluginSrcDir, "sbin"), + Rabbitmqctl = filename:join(SbinDir, "rabbitmqctl"), + RabbitmqPlugins = filename:join(SbinDir, "rabbitmq-plugins"), + RabbitmqQueues = filename:join(SbinDir, "rabbitmq-queues"), + case filelib:is_regular(Rabbitmqctl) of + true -> + ct:pal(?LOW_IMPORTANCE, + "Switching to CLI in e.g. ~s", [Rabbitmqctl]), + rabbit_ct_helpers:set_config( + Config, + [{rabbitmqctl_cmd, Rabbitmqctl}, + {rabbitmq_plugins_cmd, RabbitmqPlugins}, + {rabbitmq_queues_cmd, RabbitmqQueues}]); + false -> + Config + end. + +list_my_plugin_plugins(PluginSrcDir) -> + Files = filelib:wildcard("plugins/*", PluginSrcDir), + lists:partition( + fun(Path) -> + Filename = filename:basename(Path), + re:run(Filename, "^my_plugin-", [{capture, none}]) =:= match + end, Files). + +remove_other_plugins(PluginSrcDir, OtherPlugins) -> + ok = rabbit_file:recursive_delete( + [filename:join(PluginSrcDir, OtherPlugin) + || OtherPlugin <- OtherPlugins]). + +work_around_cli_and_rabbit_circular_dep(Config) -> + %% FIXME: We also need to copy `rabbit` in `my_plugins` plugins + %% directory, not because `my_plugin` depends on it, but because the + %% CLI erroneously depends on the broker. + %% + %% This can't be fixed easily because this is a circular dependency + %% (i.e. the broker depends on the CLI). So until a proper solution + %% is implemented, keep this second copy of the broker for the CLI + %% to find it. + InitialPluginsDir = filename:join( + ?config(current_srcdir, Config), + "plugins"), + PluginsDir = ?config(rmq_plugins_dir, Config), + lists:foreach( + fun(Path) -> + Filename = filename:basename(Path), + IsRabbit = re:run( + Filename, + "^rabbit-", [{capture, none}]) =:= match, + case IsRabbit of + true -> + Dest = filename:join(PluginsDir, Filename), + ct:pal( + ?LOW_IMPORTANCE, + "Copy `~s` to `~s` to fix CLI erroneous " + "dependency on `rabbit`", [Path, Dest]), + ok = rabbit_file:recursive_copy(Path, Dest); + false -> + ok + end + end, + filelib:wildcard(filename:join(InitialPluginsDir, "*"))), + Config. + +enable_feature_flag_on(Config, Node, FeatureName) -> + rabbit_ct_broker_helpers:rpc( + Config, Node, rabbit_feature_flags, enable, [FeatureName]). + +enable_feature_flag_everywhere(Config, FeatureName) -> + rabbit_ct_broker_helpers:rpc_all( + Config, rabbit_feature_flags, enable, [FeatureName]). + +is_feature_flag_supported(Config, FeatureName) -> + rabbit_ct_broker_helpers:rpc_all( + Config, rabbit_feature_flags, is_supported, [FeatureName]). + +is_feature_flag_enabled(Config, FeatureName) -> + rabbit_ct_broker_helpers:rpc_all( + Config, rabbit_feature_flags, is_enabled, [FeatureName]). + +is_feature_flag_subsystem_available(Config) -> + lists:all( + fun(B) -> B end, + rabbit_ct_broker_helpers:rpc_all( + Config, erlang, function_exported, [rabbit_feature_flags, list, 0])). + +feature_flags_files(Config) -> + rabbit_ct_broker_helpers:rpc_all( + Config, rabbit_feature_flags, enabled_feature_flags_list_file, []). + +log_feature_flags_of_all_nodes(Config) -> + rabbit_ct_broker_helpers:rpc_all( + Config, rabbit_feature_flags, info, [#{color => false, + lines => false}]). + +feature_flags_to_app_attrs(FeatureFlags) when is_map(FeatureFlags) -> + [{?MODULE, % Application + ?MODULE, % Module + maps:to_list(FeatureFlags)}]. + +declare_arbitrary_feature_flag(Config) -> + FeatureFlags = #{ff_from_testsuite => + #{desc => "My feature flag", + stability => stable}}, + rabbit_ct_broker_helpers:rpc_all( + Config, + rabbit_feature_flags, + inject_test_feature_flags, + [feature_flags_to_app_attrs(FeatureFlags)]), + ok. + +block(Pairs) -> [block(X, Y) || {X, Y} <- Pairs]. +unblock(Pairs) -> [allow(X, Y) || {X, Y} <- Pairs]. + +block(X, Y) -> + rabbit_ct_broker_helpers:block_traffic_between(X, Y). + +allow(X, Y) -> + rabbit_ct_broker_helpers:allow_traffic_between(X, Y). diff --git a/deps/rabbit/test/feature_flags_SUITE_data/my_plugin/.gitignore b/deps/rabbit/test/feature_flags_SUITE_data/my_plugin/.gitignore new file mode 100644 index 0000000000..f6d56e0687 --- /dev/null +++ b/deps/rabbit/test/feature_flags_SUITE_data/my_plugin/.gitignore @@ -0,0 +1,7 @@ +/.erlang.mk/ +/deps/ +/ebin/ +/escript +/plugins/ +/my_plugin.d +/sbin diff --git a/deps/rabbit/test/feature_flags_SUITE_data/my_plugin/Makefile b/deps/rabbit/test/feature_flags_SUITE_data/my_plugin/Makefile new file mode 100644 index 0000000000..8f6681090b --- /dev/null +++ b/deps/rabbit/test/feature_flags_SUITE_data/my_plugin/Makefile @@ -0,0 +1,15 @@ +PROJECT = my_plugin +PROJECT_DESCRIPTION = Plugin to test feature flags +PROJECT_VERSION = 1.0.0 + +define PROJECT_APP_EXTRA_KEYS + {broker_version_requirements, []} +endef + +DEPS = rabbit_common rabbit + +DEP_EARLY_PLUGINS = rabbit_common/mk/rabbitmq-early-plugin.mk +DEP_PLUGINS = rabbit_common/mk/rabbitmq-plugin.mk + +include rabbitmq-components.mk +include erlang.mk diff --git a/deps/rabbit/test/feature_flags_SUITE_data/my_plugin/erlang.mk b/deps/rabbit/test/feature_flags_SUITE_data/my_plugin/erlang.mk new file mode 100644 index 0000000000..f303054bad --- /dev/null +++ b/deps/rabbit/test/feature_flags_SUITE_data/my_plugin/erlang.mk @@ -0,0 +1 @@ +include ../../../erlang.mk diff --git a/deps/rabbit/test/feature_flags_SUITE_data/my_plugin/rabbitmq-components.mk b/deps/rabbit/test/feature_flags_SUITE_data/my_plugin/rabbitmq-components.mk new file mode 100644 index 0000000000..9f89dba726 --- /dev/null +++ b/deps/rabbit/test/feature_flags_SUITE_data/my_plugin/rabbitmq-components.mk @@ -0,0 +1 @@ +include ../../../rabbitmq-components.mk diff --git a/deps/rabbit/test/feature_flags_SUITE_data/my_plugin/src/my_plugin.erl b/deps/rabbit/test/feature_flags_SUITE_data/my_plugin/src/my_plugin.erl new file mode 100644 index 0000000000..687acdb5de --- /dev/null +++ b/deps/rabbit/test/feature_flags_SUITE_data/my_plugin/src/my_plugin.erl @@ -0,0 +1,10 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2019-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(my_plugin). + +-rabbit_feature_flag({plugin_ff, #{desc => "Plugin's feature flag A"}}). diff --git a/deps/rabbit/test/honeycomb_cth.erl b/deps/rabbit/test/honeycomb_cth.erl new file mode 100644 index 0000000000..dd82da13c7 --- /dev/null +++ b/deps/rabbit/test/honeycomb_cth.erl @@ -0,0 +1,105 @@ +-module(honeycomb_cth). + +-export([id/1]). +-export([init/2]). + +-export([pre_init_per_testcase/4]). +-export([post_end_per_testcase/5]). + +-record(state, {directory, github_workflow, github_run_id, + github_repository, github_sha, github_ref, + base_rmq_ref, secondary_umbrella, + erlang_version, elixir_version, + otp_release, cpu_topology, schedulers, + system_architecture, system_memory_data, + start_times = #{}}). + +id(Opts) -> + proplists:get_value(directory, Opts, "/tmp/honeycomb"). + +init(Id, _Opts) -> + application:ensure_all_started(os_mon), + {ok, #state{directory = Id, + github_workflow = os:getenv("GITHUB_WORKFLOW", "unknown"), + github_run_id = os:getenv("GITHUB_RUN_ID", "unknown"), + github_repository = os:getenv("GITHUB_REPOSITORY", "unknown"), + github_sha = os:getenv("GITHUB_SHA", "unknown"), + github_ref = os:getenv("GITHUB_REF", "unknown"), + base_rmq_ref = os:getenv("BASE_RMQ_REF", "unknown"), + secondary_umbrella = os:getenv("SECONDARY_UMBRELLA", "none"), + erlang_version = os:getenv("ERLANG_VERSION", "unknown"), + elixir_version = os:getenv("ELIXIR_VERSION", "unknown"), + otp_release = erlang:system_info(otp_release), + cpu_topology = erlang:system_info(cpu_topology), + schedulers = erlang:system_info(schedulers), + system_architecture = erlang:system_info(system_architecture), + system_memory_data = memsup:get_system_memory_data()}}. + +pre_init_per_testcase(Suite, TC, Config, #state{start_times = StartTimes} = State) -> + SuiteTimes = maps:get(Suite, StartTimes, #{}), + {Config, State#state{start_times = + StartTimes#{Suite => + SuiteTimes#{TC => erlang:timestamp()}}}}. + +post_end_per_testcase(Suite, TC, _Config, Return, #state{github_workflow = GithubWorkflow, + github_run_id = GithubRunId, + github_repository = GithubRepository, + github_sha = GithubSha, + github_ref = GithubRef, + base_rmq_ref = BaseRmqRef, + secondary_umbrella = SecondaryUmbrella, + erlang_version = ErlangVersion, + elixir_version = ElixirVersion, + otp_release = OtpRelease, + cpu_topology = CpuTopology, + schedulers = Schedulers, + system_architecture = SystemArchitecture, + system_memory_data = SystemMemoryData, + start_times = StartTimes} = State) -> + EndTime = erlang:timestamp(), + SuiteTimes = maps:get(Suite, StartTimes), + {StartTime, SuiteTimes1} = maps:take(TC, SuiteTimes), + DurationMicroseconds = timer:now_diff(EndTime, StartTime), + + File = filename(Suite, TC, State), + ok = filelib:ensure_dir(File), + {ok, F} = file:open(File, [write]), + + Json = jsx:encode([{<<"ci">>, <<"GitHub Actions">>}, + {<<"github_workflow">>, list_to_binary(GithubWorkflow)}, + {<<"github_run_id">>, list_to_binary(GithubRunId)}, + {<<"github_repository">>, list_to_binary(GithubRepository)}, + {<<"github_sha">>, list_to_binary(GithubSha)}, + {<<"github_ref">>, list_to_binary(GithubRef)}, + {<<"base_rmq_ref">>, list_to_binary(BaseRmqRef)}, + {<<"secondary_umbrella">>, list_to_binary(SecondaryUmbrella)}, + {<<"erlang_version">>, list_to_binary(ErlangVersion)}, + {<<"elixir_version">>, list_to_binary(ElixirVersion)}, + {<<"otp_release">>, list_to_binary(OtpRelease)}, + {<<"cpu_topology">>, cpu_topology_json_term(CpuTopology)}, + {<<"schedulers">>, Schedulers}, + {<<"system_architecture">>, list_to_binary(SystemArchitecture)}, + {<<"system_memory_data">>, memory_json_term(SystemMemoryData)}, + {<<"suite">>, list_to_binary(atom_to_list(Suite))}, + {<<"testcase">>, list_to_binary(atom_to_list(TC))}, + {<<"duration_seconds">>, DurationMicroseconds / 1000000}, + {<<"result">>, list_to_binary(io_lib:format("~p", [Return]))}]), + + file:write(F, Json), + file:close(F), + {Return, State#state{start_times = StartTimes#{Suite := SuiteTimes1}}}. + +filename(Suite, TC, #state{directory = Dir}) -> + filename:join(Dir, + integer_to_list(erlang:system_time()) + ++ "_" ++ atom_to_list(Suite) + ++ "_" ++ atom_to_list(TC) + ++ ".json"). + +memory_json_term(SystemMemoryData) when is_list(SystemMemoryData) -> + [{list_to_binary(atom_to_list(K)), V} || {K, V} <- SystemMemoryData]. + +cpu_topology_json_term([{processor, Cores}]) when is_list(Cores) -> + [{<<"processor">>, [begin + [{<<"core">>, [{list_to_binary(atom_to_list(Kind)), Index}]}] + end || {core, {Kind, Index}} <- Cores]}]. diff --git a/deps/rabbit/test/lazy_queue_SUITE.erl b/deps/rabbit/test/lazy_queue_SUITE.erl new file mode 100644 index 0000000000..8748b07aca --- /dev/null +++ b/deps/rabbit/test/lazy_queue_SUITE.erl @@ -0,0 +1,215 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(lazy_queue_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). + +-compile(export_all). + +-define(QNAME, <<"queue.mode.test">>). +-define(MESSAGE_COUNT, 2000). + +all() -> + [ + {group, non_parallel_tests} + ]. + +groups() -> + [ + {non_parallel_tests, [], [ + declare_args, + queue_mode_policy, + publish_consume + ]} + ]. + +%% ------------------------------------------------------------------- +%% Testsuite setup/teardown. +%% ------------------------------------------------------------------- + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(_, Config) -> + Config. + +end_per_group(_, Config) -> + Config. + +init_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_started(Config, Testcase), + ClusterSize = 2, + TestNumber = rabbit_ct_helpers:testcase_number(Config, ?MODULE, Testcase), + Config1 = rabbit_ct_helpers:set_config(Config, [ + {rmq_nodes_count, ClusterSize}, + {rmq_nodes_clustered, true}, + {rmq_nodename_suffix, Testcase}, + {tcp_ports_base, {skip_n_nodes, TestNumber * ClusterSize}} + ]), + rabbit_ct_helpers:run_steps(Config1, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps() ++ [ + fun rabbit_ct_broker_helpers:set_ha_policy_all/1 + ]). + +end_per_testcase(Testcase, Config) -> + Config1 = rabbit_ct_helpers:run_steps(Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()), + rabbit_ct_helpers:testcase_finished(Config1, Testcase). + +%% ------------------------------------------------------------------- +%% Testcases. +%% ------------------------------------------------------------------- + +declare_args(Config) -> + A = rabbit_ct_broker_helpers:get_node_config(Config, 0, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, A), + LQ = <<"lazy-q">>, + declare(Ch, LQ, [{<<"x-queue-mode">>, longstr, <<"lazy">>}]), + assert_queue_mode(A, LQ, lazy), + + DQ = <<"default-q">>, + declare(Ch, DQ, [{<<"x-queue-mode">>, longstr, <<"default">>}]), + assert_queue_mode(A, DQ, default), + + DQ2 = <<"default-q2">>, + declare(Ch, DQ2), + assert_queue_mode(A, DQ2, default), + + passed. + +queue_mode_policy(Config) -> + A = rabbit_ct_broker_helpers:get_node_config(Config, 0, nodename), + + set_ha_mode_policy(Config, A, <<"lazy">>), + + Ch = rabbit_ct_client_helpers:open_channel(Config, A), + + LQ = <<"lazy-q">>, + declare(Ch, LQ, [{<<"x-queue-mode">>, longstr, <<"lazy">>}]), + assert_queue_mode(A, LQ, lazy), + + LQ2 = <<"lazy-q-2">>, + declare(Ch, LQ2), + assert_queue_mode(A, LQ2, lazy), + + DQ = <<"default-q">>, + declare(Ch, DQ, [{<<"x-queue-mode">>, longstr, <<"default">>}]), + assert_queue_mode(A, DQ, default), + + set_ha_mode_policy(Config, A, <<"default">>), + + ok = wait_for_queue_mode(A, LQ, lazy, 5000), + ok = wait_for_queue_mode(A, LQ2, default, 5000), + ok = wait_for_queue_mode(A, DQ, default, 5000), + + passed. + +publish_consume(Config) -> + A = rabbit_ct_broker_helpers:get_node_config(Config, 0, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, A), + declare(Ch, ?QNAME), + + rabbit_ct_client_helpers:publish(Ch, ?QNAME, ?MESSAGE_COUNT), + consume(Ch, ?QNAME, ack), + [assert_delivered(Ch, ack, P) || P <- lists:seq(1, ?MESSAGE_COUNT)], + + set_ha_mode_policy(Config, A, <<"lazy">>), + rabbit_ct_client_helpers:publish(Ch, ?QNAME, ?MESSAGE_COUNT), + rabbit_ct_client_helpers:publish(Ch, ?QNAME, ?MESSAGE_COUNT), + [assert_delivered(Ch, ack, P) || P <- lists:seq(1, ?MESSAGE_COUNT)], + + set_ha_mode_policy(Config, A, <<"default">>), + [assert_delivered(Ch, ack, P) || P <- lists:seq(1, ?MESSAGE_COUNT)], + + rabbit_ct_client_helpers:publish(Ch, ?QNAME, ?MESSAGE_COUNT), + set_ha_mode_policy(Config, A, <<"lazy">>), + rabbit_ct_client_helpers:publish(Ch, ?QNAME, ?MESSAGE_COUNT), + set_ha_mode_policy(Config, A, <<"default">>), + [assert_delivered(Ch, ack, P) || P <- lists:seq(1, ?MESSAGE_COUNT)], + + set_ha_mode_policy(Config, A, <<"lazy">>), + [assert_delivered(Ch, ack, P) || P <- lists:seq(1, ?MESSAGE_COUNT)], + + cancel(Ch), + + passed. + +%%---------------------------------------------------------------------------- + +declare(Ch, Q) -> + declare(Ch, Q, []). + +declare(Ch, Q, Args) -> + amqp_channel:call(Ch, #'queue.declare'{queue = Q, + durable = true, + arguments = Args}). + +consume(Ch, Q, Ack) -> + amqp_channel:subscribe(Ch, #'basic.consume'{queue = Q, + no_ack = Ack =:= no_ack, + consumer_tag = <<"ctag">>}, + self()), + receive + #'basic.consume_ok'{consumer_tag = <<"ctag">>} -> + ok + end. + +cancel(Ch) -> + amqp_channel:call(Ch, #'basic.cancel'{consumer_tag = <<"ctag">>}). + +assert_delivered(Ch, Ack, Payload) -> + PBin = payload2bin(Payload), + receive + {#'basic.deliver'{delivery_tag = DTag}, #amqp_msg{payload = PBin2}} -> + PBin = PBin2, + maybe_ack(Ch, Ack, DTag) + end. + +maybe_ack(Ch, do_ack, DTag) -> + amqp_channel:cast(Ch, #'basic.ack'{delivery_tag = DTag}), + DTag; +maybe_ack(_Ch, _, DTag) -> + DTag. + +payload2bin(Int) -> list_to_binary(integer_to_list(Int)). + +set_ha_mode_policy(Config, Node, Mode) -> + ok = rabbit_ct_broker_helpers:set_ha_policy(Config, Node, <<".*">>, <<"all">>, + [{<<"queue-mode">>, Mode}]). + + +wait_for_queue_mode(_Node, _Q, _Mode, Max) when Max < 0 -> + fail; +wait_for_queue_mode(Node, Q, Mode, Max) -> + case get_queue_mode(Node, Q) of + Mode -> ok; + _ -> timer:sleep(100), + wait_for_queue_mode(Node, Q, Mode, Max - 100) + end. + +assert_queue_mode(Node, Q, Expected) -> + Actual = get_queue_mode(Node, Q), + Expected = Actual. + +get_queue_mode(Node, Q) -> + QNameRes = rabbit_misc:r(<<"/">>, queue, Q), + {ok, AMQQueue} = + rpc:call(Node, rabbit_amqqueue, lookup, [QNameRes]), + [{backing_queue_status, Status}] = + rpc:call(Node, rabbit_amqqueue, info, + [AMQQueue, [backing_queue_status]]), + proplists:get_value(mode, Status). diff --git a/deps/rabbit/test/list_consumers_sanity_check_SUITE.erl b/deps/rabbit/test/list_consumers_sanity_check_SUITE.erl new file mode 100644 index 0000000000..fbd31fa3e8 --- /dev/null +++ b/deps/rabbit/test/list_consumers_sanity_check_SUITE.erl @@ -0,0 +1,125 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2011-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(list_consumers_sanity_check_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). + +-compile(export_all). + +all() -> + [ + {group, list_consumers_sanity_check} + ]. + +groups() -> + [ + {list_consumers_sanity_check, [], [ + list_consumers_sanity_check + ]} + ]. + +group(_) -> + []. + +%% ------------------------------------------------------------------- +%% Testsuite setup/teardown. +%% ------------------------------------------------------------------- + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(Group, Config) -> + Config1 = rabbit_ct_helpers:set_config(Config, [ + {rmq_nodename_suffix, Group}, + {rmq_nodes_count, 1} + ]), + rabbit_ct_helpers:run_steps(Config1, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()). + +end_per_group(_Group, Config) -> + rabbit_ct_helpers:run_steps(Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()). + +init_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_started(Config, Testcase). + +end_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_finished(Config, Testcase). + +%% ------------------------------------------------------------------- +%% Testcase +%% ------------------------------------------------------------------- + +list_consumers_sanity_check(Config) -> + A = rabbit_ct_broker_helpers:get_node_config(Config, 0, nodename), + Chan = rabbit_ct_client_helpers:open_channel(Config, A), + %% this queue is not cleaned up because the entire node is + %% reset between tests + QName = <<"list_consumers_q">>, + #'queue.declare_ok'{} = amqp_channel:call(Chan, #'queue.declare'{queue = QName}), + + %% No consumers even if we have some queues + [] = rabbitmqctl_list_consumers(Config, A), + + %% Several consumers on single channel should be correctly reported + #'basic.consume_ok'{consumer_tag = CTag1} = amqp_channel:call(Chan, #'basic.consume'{queue = QName}), + #'basic.consume_ok'{consumer_tag = CTag2} = amqp_channel:call(Chan, #'basic.consume'{queue = QName}), + true = (lists:sort([CTag1, CTag2]) =:= + lists:sort(rabbitmqctl_list_consumers(Config, A))), + + %% `rabbitmqctl report` shares some code with `list_consumers`, so + %% check that it also reports both channels + {ok, ReportStdOut} = rabbit_ct_broker_helpers:rabbitmqctl(Config, A, + ["list_consumers", "--no-table-headers"]), + ReportLines = re:split(ReportStdOut, <<"\n">>, [trim]), + ReportCTags = [lists:nth(3, re:split(Row, <<"\t">>)) || <<"list_consumers_q", _/binary>> = Row <- ReportLines], + true = (lists:sort([CTag1, CTag2]) =:= + lists:sort(ReportCTags)). + +rabbitmqctl_list_consumers(Config, Node) -> + {ok, StdOut} = rabbit_ct_broker_helpers:rabbitmqctl(Config, Node, + ["list_consumers", "--no-table-headers"]), + [<<"Listing consumers", _/binary>> | ConsumerRows] = re:split(StdOut, <<"\n">>, [trim]), + CTags = [ lists:nth(3, re:split(Row, <<"\t">>)) || Row <- ConsumerRows ], + CTags. + +list_queues_online_and_offline(Config) -> + [A, B] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + ACh = rabbit_ct_client_helpers:open_channel(Config, A), + %% Node B will be stopped + BCh = rabbit_ct_client_helpers:open_channel(Config, B), + #'queue.declare_ok'{} = amqp_channel:call(ACh, #'queue.declare'{queue = <<"q_a_1">>, durable = true}), + #'queue.declare_ok'{} = amqp_channel:call(ACh, #'queue.declare'{queue = <<"q_a_2">>, durable = true}), + #'queue.declare_ok'{} = amqp_channel:call(BCh, #'queue.declare'{queue = <<"q_b_1">>, durable = true}), + #'queue.declare_ok'{} = amqp_channel:call(BCh, #'queue.declare'{queue = <<"q_b_2">>, durable = true}), + + rabbit_ct_broker_helpers:rabbitmqctl(Config, B, ["stop"]), + + GotUp = lists:sort(rabbit_ct_broker_helpers:rabbitmqctl_list(Config, A, + ["list_queues", "--online", "name", "--no-table-headers"])), + ExpectUp = [[<<"q_a_1">>], [<<"q_a_2">>]], + ExpectUp = GotUp, + + GotDown = lists:sort(rabbit_ct_broker_helpers:rabbitmqctl_list(Config, A, + ["list_queues", "--offline", "name", "--no-table-headers"])), + ExpectDown = [[<<"q_b_1">>], [<<"q_b_2">>]], + ExpectDown = GotDown, + + GotAll = lists:sort(rabbit_ct_broker_helpers:rabbitmqctl_list(Config, A, + ["list_queues", "name", "--no-table-headers"])), + ExpectAll = ExpectUp ++ ExpectDown, + ExpectAll = GotAll, + + ok. diff --git a/deps/rabbit/test/list_queues_online_and_offline_SUITE.erl b/deps/rabbit/test/list_queues_online_and_offline_SUITE.erl new file mode 100644 index 0000000000..d26fdc03e2 --- /dev/null +++ b/deps/rabbit/test/list_queues_online_and_offline_SUITE.erl @@ -0,0 +1,99 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2011-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(list_queues_online_and_offline_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). + +-compile(export_all). + +all() -> + [ + {group, list_queues_online_and_offline} + ]. + +groups() -> + [ + {list_queues_online_and_offline, [], [ + list_queues_online_and_offline %% Stop node B. + ]} + ]. + +group(_) -> + []. + +%% ------------------------------------------------------------------- +%% Testsuite setup/teardown. +%% ------------------------------------------------------------------- + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(Group, Config) -> + Config1 = rabbit_ct_helpers:set_config(Config, + [ + {rmq_nodename_suffix, Group}, + {rmq_nodes_count, 2} + ]), + rabbit_ct_helpers:run_steps( + Config1, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()). + +end_per_group(_Group, Config) -> + rabbit_ct_helpers:run_steps(Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()). + +init_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_started(Config, Testcase). + +end_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_finished(Config, Testcase). + +%% --------------------------------------------------------------------------- +%% Testcase +%% --------------------------------------------------------------------------- + +list_queues_online_and_offline(Config) -> + [A, B] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + ACh = rabbit_ct_client_helpers:open_channel(Config, A), + %% Node B will be stopped + BCh = rabbit_ct_client_helpers:open_channel(Config, B), + #'queue.declare_ok'{} = amqp_channel:call(ACh, #'queue.declare'{queue = <<"q_a_1">>, durable = true}), + #'queue.declare_ok'{} = amqp_channel:call(ACh, #'queue.declare'{queue = <<"q_a_2">>, durable = true}), + #'queue.declare_ok'{} = amqp_channel:call(BCh, #'queue.declare'{queue = <<"q_b_1">>, durable = true}), + #'queue.declare_ok'{} = amqp_channel:call(BCh, #'queue.declare'{queue = <<"q_b_2">>, durable = true}), + + rabbit_ct_broker_helpers:rabbitmqctl(Config, B, ["stop"]), + + rabbit_ct_helpers:await_condition( + fun() -> + [A] == rpc:call(A, rabbit_mnesia, cluster_nodes, [running]) + end, 60000), + + GotUp = lists:sort(rabbit_ct_broker_helpers:rabbitmqctl_list(Config, A, + ["list_queues", "--online", "name", "--no-table-headers"])), + ExpectUp = [[<<"q_a_1">>], [<<"q_a_2">>]], + ExpectUp = GotUp, + + GotDown = lists:sort(rabbit_ct_broker_helpers:rabbitmqctl_list(Config, A, + ["list_queues", "--offline", "name", "--no-table-headers"])), + ExpectDown = [[<<"q_b_1">>], [<<"q_b_2">>]], + ExpectDown = GotDown, + + GotAll = lists:sort(rabbit_ct_broker_helpers:rabbitmqctl_list(Config, A, + ["list_queues", "name", "--no-table-headers"])), + ExpectAll = ExpectUp ++ ExpectDown, + ExpectAll = GotAll, + + ok. diff --git a/deps/rabbit/test/maintenance_mode_SUITE.erl b/deps/rabbit/test/maintenance_mode_SUITE.erl new file mode 100644 index 0000000000..3abbf9b064 --- /dev/null +++ b/deps/rabbit/test/maintenance_mode_SUITE.erl @@ -0,0 +1,284 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(maintenance_mode_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-compile(export_all). + +all() -> + [ + {group, cluster_size_3}, + {group, quorum_queues} + ]. + +groups() -> + [ + {cluster_size_3, [], [ + maintenance_mode_status, + listener_suspension_status, + client_connection_closure, + classic_mirrored_queue_leadership_transfer + ]}, + {quorum_queues, [], [ + quorum_queue_leadership_transfer + ]} + ]. + +%% ------------------------------------------------------------------- +%% Setup and teardown. +%% ------------------------------------------------------------------- + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(_Group, Config) -> + rabbit_ct_helpers:set_config(Config, [ + {rmq_nodes_count, 3} + ]). + +end_per_group(_, Config) -> + Config. + +init_per_testcase(quorum_queue_leadership_transfer = Testcase, Config) -> + rabbit_ct_helpers:testcase_started(Config, Testcase), + ClusterSize = ?config(rmq_nodes_count, Config), + TestNumber = rabbit_ct_helpers:testcase_number(Config, ?MODULE, Testcase), + Config1 = rabbit_ct_helpers:set_config(Config, [ + {rmq_nodes_clustered, true}, + {rmq_nodename_suffix, Testcase}, + {tcp_ports_base, {skip_n_nodes, TestNumber * ClusterSize}} + ]), + Config2 = rabbit_ct_helpers:run_steps( + Config1, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()), + MaintenanceModeFFEnabled = rabbit_ct_broker_helpers:enable_feature_flag( + Config2, maintenance_mode_status), + QuorumQueueFFEnabled = rabbit_ct_broker_helpers:enable_feature_flag( + Config2, quorum_queue), + case MaintenanceModeFFEnabled of + ok -> + case QuorumQueueFFEnabled of + ok -> + Config2; + Skip -> + end_per_testcase(Testcase, Config2), + Skip + end; + Skip -> + end_per_testcase(Testcase, Config2), + Skip + end; +init_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_started(Config, Testcase), + ClusterSize = ?config(rmq_nodes_count, Config), + TestNumber = rabbit_ct_helpers:testcase_number(Config, ?MODULE, Testcase), + Config1 = rabbit_ct_helpers:set_config(Config, [ + {rmq_nodes_clustered, true}, + {rmq_nodename_suffix, Testcase}, + {tcp_ports_base, {skip_n_nodes, TestNumber * ClusterSize}} + ]), + Config2 = rabbit_ct_helpers:run_steps( + Config1, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps() ++ + [fun rabbit_ct_broker_helpers:set_ha_policy_all/1]), + MaintenanceModeFFEnabled = rabbit_ct_broker_helpers:enable_feature_flag( + Config2, + maintenance_mode_status), + case MaintenanceModeFFEnabled of + ok -> + Config2; + Skip -> + end_per_testcase(Testcase, Config2), + Skip + end. + +end_per_testcase(Testcase, Config) -> + Config1 = rabbit_ct_helpers:run_steps(Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()), + rabbit_ct_helpers:testcase_finished(Config1, Testcase). + +%% ------------------------------------------------------------------- +%% Test Cases +%% ------------------------------------------------------------------- + +maintenance_mode_status(Config) -> + Nodes = [A, B, C] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + [begin + ?assertNot(rabbit_ct_broker_helpers:is_being_drained_local_read(Config, Node)), + ?assertNot(rabbit_ct_broker_helpers:is_being_drained_consistent_read(Config, Node)) + end || Node <- Nodes], + + [begin + [begin + ?assertNot(rabbit_ct_broker_helpers:is_being_drained_consistent_read(Config, TargetNode, NodeToCheck)) + end || NodeToCheck <- Nodes] + end || TargetNode <- Nodes], + + rabbit_ct_broker_helpers:mark_as_being_drained(Config, B), + rabbit_ct_helpers:await_condition( + fun () -> rabbit_ct_broker_helpers:is_being_drained_local_read(Config, B) end, + 10000), + + [begin + ?assert(rabbit_ct_broker_helpers:is_being_drained_consistent_read(Config, TargetNode, B)) + end || TargetNode <- Nodes], + + ?assertEqual( + lists:usort([A, C]), + lists:usort(rabbit_ct_broker_helpers:rpc(Config, B, + rabbit_maintenance, primary_replica_transfer_candidate_nodes, []))), + + rabbit_ct_broker_helpers:unmark_as_being_drained(Config, B), + rabbit_ct_helpers:await_condition( + fun () -> not rabbit_ct_broker_helpers:is_being_drained_local_read(Config, B) end, + 10000), + + [begin + ?assertNot(rabbit_ct_broker_helpers:is_being_drained_local_read(Config, TargetNode, B)), + ?assertNot(rabbit_ct_broker_helpers:is_being_drained_consistent_read(Config, TargetNode, B)) + end || TargetNode <- Nodes], + + ?assertEqual( + lists:usort([A, C]), + lists:usort(rabbit_ct_broker_helpers:rpc(Config, B, + rabbit_maintenance, primary_replica_transfer_candidate_nodes, []))), + + ok. + + +listener_suspension_status(Config) -> + Nodes = [A | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + ct:pal("Picked node ~s for maintenance tests...", [A]), + + rabbit_ct_helpers:await_condition( + fun () -> not rabbit_ct_broker_helpers:is_being_drained_local_read(Config, A) end, 10000), + + [begin + ?assertNot(rabbit_ct_broker_helpers:is_being_drained_consistent_read(Config, Node)) + end || Node <- Nodes], + + Conn1 = rabbit_ct_client_helpers:open_connection(Config, A), + ?assert(is_pid(Conn1)), + rabbit_ct_client_helpers:close_connection(Conn1), + + rabbit_ct_broker_helpers:drain_node(Config, A), + rabbit_ct_helpers:await_condition( + fun () -> rabbit_ct_broker_helpers:is_being_drained_local_read(Config, A) end, 10000), + + ?assertEqual({error, econnrefused}, rabbit_ct_client_helpers:open_unmanaged_connection(Config, A)), + + rabbit_ct_broker_helpers:revive_node(Config, A), + rabbit_ct_helpers:await_condition( + fun () -> not rabbit_ct_broker_helpers:is_being_drained_local_read(Config, A) end, 10000), + + Conn3 = rabbit_ct_client_helpers:open_connection(Config, A), + ?assert(is_pid(Conn3)), + rabbit_ct_client_helpers:close_connection(Conn3), + + ok. + + +client_connection_closure(Config) -> + [A | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + ct:pal("Picked node ~s for maintenance tests...", [A]), + + rabbit_ct_helpers:await_condition( + fun () -> not rabbit_ct_broker_helpers:is_being_drained_local_read(Config, A) end, 10000), + + Conn1 = rabbit_ct_client_helpers:open_connection(Config, A), + ?assert(is_pid(Conn1)), + ?assertEqual(1, length(rabbit_ct_broker_helpers:rpc(Config, A, rabbit_networking, local_connections, []))), + + rabbit_ct_broker_helpers:drain_node(Config, A), + ?assertEqual(0, length(rabbit_ct_broker_helpers:rpc(Config, A, rabbit_networking, local_connections, []))), + + rabbit_ct_broker_helpers:revive_node(Config, A). + + +classic_mirrored_queue_leadership_transfer(Config) -> + [A | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + ct:pal("Picked node ~s for maintenance tests...", [A]), + + rabbit_ct_helpers:await_condition( + fun () -> not rabbit_ct_broker_helpers:is_being_drained_local_read(Config, A) end, 10000), + + PolicyPattern = <<"^cq.mirrored">>, + rabbit_ct_broker_helpers:set_ha_policy(Config, A, PolicyPattern, <<"all">>), + + Conn = rabbit_ct_client_helpers:open_connection(Config, A), + {ok, Ch} = amqp_connection:open_channel(Conn), + QName = <<"cq.mirrored.1">>, + amqp_channel:call(Ch, #'queue.declare'{queue = QName, durable = true}), + + ?assertEqual(1, length(rabbit_ct_broker_helpers:rpc(Config, A, rabbit_amqqueue, list_local, [<<"/">>]))), + + rabbit_ct_broker_helpers:drain_node(Config, A), + rabbit_ct_helpers:await_condition( + fun () -> rabbit_ct_broker_helpers:is_being_drained_local_read(Config, A) end, 10000), + + ?assertEqual(0, length(rabbit_ct_broker_helpers:rpc(Config, A, rabbit_amqqueue, list_local, [<<"/">>]))), + + rabbit_ct_broker_helpers:revive_node(Config, A), + %% rabbit_ct_broker_helpers:set_ha_policy/4 uses pattern for policy name + rabbit_ct_broker_helpers:clear_policy(Config, A, PolicyPattern). + +quorum_queue_leadership_transfer(Config) -> + [A | _] = Nodenames = rabbit_ct_broker_helpers:get_node_configs( + Config, nodename), + ct:pal("Picked node ~s for maintenance tests...", [A]), + + rabbit_ct_helpers:await_condition( + fun () -> not rabbit_ct_broker_helpers:is_being_drained_local_read(Config, A) end, 10000), + + Conn = rabbit_ct_client_helpers:open_connection(Config, A), + {ok, Ch} = amqp_connection:open_channel(Conn), + QName = <<"qq.1">>, + amqp_channel:call(Ch, #'queue.declare'{queue = QName, durable = true, arguments = [ + {<<"x-queue-type">>, longstr, <<"quorum">>} + ]}), + + %% we cannot assert on the number of local leaders here: declaring a QQ on node A + %% does not guarantee that the leader will be hosted on node A + + rabbit_ct_broker_helpers:drain_node(Config, A), + rabbit_ct_helpers:await_condition( + fun () -> rabbit_ct_broker_helpers:is_being_drained_local_read(Config, A) end, 10000), + + %% quorum queue leader election is asynchronous + AllTheSame = quorum_queue_utils:fifo_machines_use_same_version( + Config, Nodenames), + case AllTheSame of + true -> + rabbit_ct_helpers:await_condition( + fun () -> + LocalLeaders = rabbit_ct_broker_helpers:rpc( + Config, A, + rabbit_amqqueue, + list_local_leaders, + []), + length(LocalLeaders) =:= 0 + end, 20000); + false -> + ct:pal( + ?LOW_IMPORTANCE, + "Skip leader election check because rabbit_fifo machines " + "have different versions", []) + end, + + rabbit_ct_broker_helpers:revive_node(Config, A). diff --git a/deps/rabbit/test/many_node_ha_SUITE.erl b/deps/rabbit/test/many_node_ha_SUITE.erl new file mode 100644 index 0000000000..ece7dc8830 --- /dev/null +++ b/deps/rabbit/test/many_node_ha_SUITE.erl @@ -0,0 +1,112 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(many_node_ha_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). + +-compile(export_all). + +suite() -> + [ + {timetrap, {minutes, 5}} + ]. + +all() -> + [ + {group, cluster_size_6} + ]. + +groups() -> + [ + {cluster_size_6, [], [ + kill_intermediate + ]} + ]. + +%% ------------------------------------------------------------------- +%% Testsuite setup/teardown. +%% ------------------------------------------------------------------- + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(cluster_size_6, Config) -> + rabbit_ct_helpers:set_config(Config, [ + {rmq_nodes_count, 6} + ]). + +end_per_group(_, Config) -> + Config. + +init_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_started(Config, Testcase), + ClusterSize = ?config(rmq_nodes_count, Config), + TestNumber = rabbit_ct_helpers:testcase_number(Config, ?MODULE, Testcase), + Config1 = rabbit_ct_helpers:set_config(Config, [ + {rmq_nodes_clustered, true}, + {rmq_nodename_suffix, Testcase}, + {tcp_ports_base, {skip_n_nodes, TestNumber * ClusterSize}} + ]), + rabbit_ct_helpers:run_steps(Config1, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps() ++ [ + fun rabbit_ct_broker_helpers:set_ha_policy_all/1 + ]). + +end_per_testcase(Testcase, Config) -> + Config1 = rabbit_ct_helpers:run_steps(Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()), + rabbit_ct_helpers:testcase_finished(Config1, Testcase). + +%% ------------------------------------------------------------------- +%% Test Cases +%% ------------------------------------------------------------------- + +kill_intermediate(Config) -> + [A, B, C, D, E, F] = rabbit_ct_broker_helpers:get_node_configs(Config, + nodename), + Msgs = rabbit_ct_helpers:cover_work_factor(Config, 20000), + MasterChannel = rabbit_ct_client_helpers:open_channel(Config, A), + ConsumerChannel = rabbit_ct_client_helpers:open_channel(Config, E), + ProducerChannel = rabbit_ct_client_helpers:open_channel(Config, F), + Queue = <<"test">>, + amqp_channel:call(MasterChannel, #'queue.declare'{queue = Queue, + auto_delete = false}), + + %% TODO: this seems *highly* timing dependant - the assumption being + %% that the kill will work quickly enough that there will still be + %% some messages in-flight that we *must* receive despite the intervening + %% node deaths. It would be nice if we could find a means to do this + %% in a way that is not actually timing dependent. + + %% Worse still, it assumes that killing the master will cause a + %% failover to Slave1, and so on. Nope. + + ConsumerPid = rabbit_ha_test_consumer:create(ConsumerChannel, + Queue, self(), false, Msgs), + + ProducerPid = rabbit_ha_test_producer:create(ProducerChannel, + Queue, self(), false, Msgs), + + %% create a killer for the master and the first 3 mirrors + [rabbit_ct_broker_helpers:kill_node_after(Config, Node, Time) || + {Node, Time} <- [{A, 50}, + {B, 50}, + {C, 100}, + {D, 100}]], + + %% verify that the consumer got all msgs, or die, or time out + rabbit_ha_test_producer:await_response(ProducerPid), + rabbit_ha_test_consumer:await_response(ConsumerPid), + ok. diff --git a/deps/rabbit/test/message_size_limit_SUITE.erl b/deps/rabbit/test/message_size_limit_SUITE.erl new file mode 100644 index 0000000000..f43a582c85 --- /dev/null +++ b/deps/rabbit/test/message_size_limit_SUITE.erl @@ -0,0 +1,145 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2011-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(message_size_limit_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("kernel/include/file.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-compile(export_all). + +-define(TIMEOUT_LIST_OPS_PASS, 5000). +-define(TIMEOUT, 30000). +-define(TIMEOUT_CHANNEL_EXCEPTION, 5000). + +-define(CLEANUP_QUEUE_NAME, <<"cleanup-queue">>). + +all() -> + [ + {group, parallel_tests} + ]. + +groups() -> + [ + {parallel_tests, [parallel], [ + max_message_size + ]} + ]. + +suite() -> + [ + {timetrap, {minutes, 3}} + ]. + +%% ------------------------------------------------------------------- +%% Testsuite setup/teardown. +%% ------------------------------------------------------------------- + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(Group, Config) -> + Config1 = rabbit_ct_helpers:set_config(Config, [ + {rmq_nodename_suffix, Group}, + {rmq_nodes_count, 1} + ]), + rabbit_ct_helpers:run_steps(Config1, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()). + +end_per_group(_Group, Config) -> + rabbit_ct_helpers:run_steps(Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()). + +init_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_started(Config, Testcase). + +end_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_finished(Config, Testcase). + +%% ------------------------------------------------------------------- +%% Test cases +%% ------------------------------------------------------------------- + +max_message_size(Config) -> + Binary2M = gen_binary_mb(2), + Binary4M = gen_binary_mb(4), + Binary6M = gen_binary_mb(6), + Binary10M = gen_binary_mb(10), + + Size2Mb = 1024 * 1024 * 2, + Size2Mb = byte_size(Binary2M), + + rabbit_ct_broker_helpers:rpc(Config, 0, + application, set_env, [rabbit, max_message_size, 1024 * 1024 * 3]), + + {_, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + + %% Binary is within the max size limit + amqp_channel:call(Ch, #'basic.publish'{routing_key = <<"none">>}, #amqp_msg{payload = Binary2M}), + %% The channel process is alive + assert_channel_alive(Ch), + + Monitor = monitor(process, Ch), + amqp_channel:call(Ch, #'basic.publish'{routing_key = <<"none">>}, #amqp_msg{payload = Binary4M}), + assert_channel_fail_max_size(Ch, Monitor), + + %% increase the limit + rabbit_ct_broker_helpers:rpc(Config, 0, + application, set_env, [rabbit, max_message_size, 1024 * 1024 * 8]), + + {_, Ch1} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + + amqp_channel:call(Ch1, #'basic.publish'{routing_key = <<"nope">>}, #amqp_msg{payload = Binary2M}), + assert_channel_alive(Ch1), + + amqp_channel:call(Ch1, #'basic.publish'{routing_key = <<"nope">>}, #amqp_msg{payload = Binary4M}), + assert_channel_alive(Ch1), + + amqp_channel:call(Ch1, #'basic.publish'{routing_key = <<"nope">>}, #amqp_msg{payload = Binary6M}), + assert_channel_alive(Ch1), + + Monitor1 = monitor(process, Ch1), + amqp_channel:call(Ch1, #'basic.publish'{routing_key = <<"none">>}, #amqp_msg{payload = Binary10M}), + assert_channel_fail_max_size(Ch1, Monitor1), + + %% increase beyond the hard limit + rabbit_ct_broker_helpers:rpc(Config, 0, + application, set_env, [rabbit, max_message_size, 1024 * 1024 * 600]), + Val = rabbit_ct_broker_helpers:rpc(Config, 0, + rabbit_channel, get_max_message_size, []), + + ?assertEqual(?MAX_MSG_SIZE, Val). + +%% ------------------------------------------------------------------- +%% Implementation +%% ------------------------------------------------------------------- + +gen_binary_mb(N) -> + B1M = << <<"_">> || _ <- lists:seq(1, 1024 * 1024) >>, + << B1M || _ <- lists:seq(1, N) >>. + +assert_channel_alive(Ch) -> + amqp_channel:call(Ch, #'basic.publish'{routing_key = <<"nope">>}, + #amqp_msg{payload = <<"HI">>}). + +assert_channel_fail_max_size(Ch, Monitor) -> + receive + {'DOWN', Monitor, process, Ch, + {shutdown, + {server_initiated_close, 406, _Error}}} -> + ok + after ?TIMEOUT_CHANNEL_EXCEPTION -> + error({channel_exception_expected, max_message_size}) + end. diff --git a/deps/rabbit/test/metrics_SUITE.erl b/deps/rabbit/test/metrics_SUITE.erl new file mode 100644 index 0000000000..e585ccd5a8 --- /dev/null +++ b/deps/rabbit/test/metrics_SUITE.erl @@ -0,0 +1,404 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2016-2020 VMware, Inc. or its affiliates. All rights reserved. +%% +-module(metrics_SUITE). +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("proper/include/proper.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). +-include_lib("rabbit_common/include/rabbit_core_metrics.hrl"). +-include_lib("rabbitmq_ct_helpers/include/rabbit_assert.hrl"). + + +all() -> + [ + {group, non_parallel_tests} + ]. + +groups() -> + [ + {non_parallel_tests, [], [ + connection, + channel, + channel_connection_close, + channel_queue_exchange_consumer_close_connection, + channel_queue_delete_queue, + connection_metric_count_test, + channel_metric_count_test, + queue_metric_count_test, + queue_metric_count_channel_per_queue_test, + connection_metric_idemp_test, + channel_metric_idemp_test, + queue_metric_idemp_test + ]} + ]. + +%% ------------------------------------------------------------------- +%% Testsuite setup/teardown. +%% ------------------------------------------------------------------- + +merge_app_env(Config) -> + rabbit_ct_helpers:merge_app_env(Config, + {rabbit, [ + {collect_statistics, fine}, + {collect_statistics_interval, 500} + ]}). +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + Config1 = rabbit_ct_helpers:set_config(Config, [ + {rmq_nodename_suffix, ?MODULE} + ]), + rabbit_ct_helpers:run_setup_steps(Config1, + [ fun merge_app_env/1 ] ++ + rabbit_ct_broker_helpers:setup_steps()). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config, + rabbit_ct_broker_helpers:teardown_steps()). + +init_per_group(_, Config) -> + Config. + +end_per_group(_, Config) -> + Config. + +init_per_testcase(Testcase, Config) -> + clean_core_metrics(Config), + rabbit_ct_helpers:testcase_started(Config, Testcase). + +end_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_finished(Config, Testcase). + + +%% ------------------------------------------------------------------- +%% Testcases. +%% ------------------------------------------------------------------- + +% NB: node_stats tests are in the management_agent repo + +connection_metric_count_test(Config) -> + rabbit_ct_proper_helpers:run_proper(fun prop_connection_metric_count/1, [Config], 25). + +channel_metric_count_test(Config) -> + rabbit_ct_proper_helpers:run_proper(fun prop_channel_metric_count/1, [Config], 25). + +queue_metric_count_test(Config) -> + rabbit_ct_proper_helpers:run_proper(fun prop_queue_metric_count/1, [Config], 5). + +queue_metric_count_channel_per_queue_test(Config) -> + rabbit_ct_proper_helpers:run_proper(fun prop_queue_metric_count_channel_per_queue/1, + [Config], 5). + +connection_metric_idemp_test(Config) -> + connection_metric_idemp(Config, {1, 1}), + connection_metric_idemp(Config, {1, 2}), + connection_metric_idemp(Config, {2, 2}). + +channel_metric_idemp_test(Config) -> + rabbit_ct_proper_helpers:run_proper(fun prop_channel_metric_idemp/1, [Config], 25). + +queue_metric_idemp_test(Config) -> + rabbit_ct_proper_helpers:run_proper(fun prop_queue_metric_idemp/1, [Config], 25). + +prop_connection_metric_idemp(Config) -> + ?FORALL(N, {integer(1, 25), integer(1, 25)}, + connection_metric_idemp(Config, N)). + +prop_channel_metric_idemp(Config) -> + ?FORALL(N, {integer(1, 25), integer(1, 25)}, + channel_metric_idemp(Config, N)). + +prop_queue_metric_idemp(Config) -> + ?FORALL(N, {integer(1, 25), integer(1, 25)}, + queue_metric_idemp(Config, N)). + +prop_connection_metric_count(Config) -> + ?FORALL(N, {integer(1, 25), resize(100, list(oneof([add, remove])))}, + connection_metric_count(Config, N)). + +prop_channel_metric_count(Config) -> + ?FORALL(N, {integer(1, 25), resize(100, list(oneof([add, remove])))}, + channel_metric_count(Config, N)). + +prop_queue_metric_count(Config) -> + ?FORALL(N, {integer(1, 10), resize(10, list(oneof([add, remove])))}, + queue_metric_count(Config, N)). + +prop_queue_metric_count_channel_per_queue(Config) -> + ?FORALL(N, {integer(1, 10), resize(10, list(oneof([add, remove])))}, + queue_metric_count_channel_per_queue(Config, N)). + +connection_metric_idemp(Config, {N, R}) -> + Conns = [rabbit_ct_client_helpers:open_unmanaged_connection(Config) + || _ <- lists:seq(1, N)], + Table = ?awaitMatch(L when is_list(L) andalso length(L) == N, + [ Pid || {Pid, _} <- read_table_rpc(Config, + connection_metrics)], + 5000), + Table2 = [ Pid || {Pid, _} <- read_table_rpc(Config, connection_coarse_metrics)], + % refresh stats 'R' times + [[Pid ! emit_stats || Pid <- Table] || _ <- lists:seq(1, R)], + force_metric_gc(Config), + TableAfter = [ Pid || {Pid, _} <- read_table_rpc(Config, connection_metrics)], + TableAfter2 = [ Pid || {Pid, _} <- read_table_rpc(Config, connection_coarse_metrics)], + [rabbit_ct_client_helpers:close_connection(Conn) || Conn <- Conns], + ?assertEqual(Table, TableAfter), + ?assertEqual(Table2, TableAfter2), + ?assertEqual(N, length(Table)), + ?assertEqual(N, length(TableAfter)). + +channel_metric_idemp(Config, {N, R}) -> + Conn = rabbit_ct_client_helpers:open_unmanaged_connection(Config), + [amqp_connection:open_channel(Conn) || _ <- lists:seq(1, N)], + Table = [ Pid || {Pid, _} <- read_table_rpc(Config, channel_metrics)], + Table2 = [ Pid || {Pid, _} <- read_table_rpc(Config, channel_process_metrics)], + % refresh stats 'R' times + [[Pid ! emit_stats || Pid <- Table] || _ <- lists:seq(1, R)], + force_metric_gc(Config), + TableAfter = [ Pid || {Pid, _} <- read_table_rpc(Config, channel_metrics)], + TableAfter2 = [ Pid || {Pid, _} <- read_table_rpc(Config, channel_process_metrics)], + rabbit_ct_client_helpers:close_connection(Conn), + (Table2 == TableAfter2) and (Table == TableAfter) and + (N == length(Table)) and (N == length(TableAfter)). + +queue_metric_idemp(Config, {N, R}) -> + clean_core_metrics(Config), + Conn = rabbit_ct_client_helpers:open_unmanaged_connection(Config), + {ok, Chan} = amqp_connection:open_channel(Conn), + Queues = + [begin + Queue = declare_queue(Chan), + ensure_exchange_metrics_populated(Chan, Queue), + ensure_channel_queue_metrics_populated(Chan, Queue), + Queue + end || _ <- lists:seq(1, N)], + + Table = [ Pid || {Pid, _, _} <- read_table_rpc(Config, queue_metrics)], + Table2 = [ Pid || {Pid, _, _} <- read_table_rpc(Config, queue_coarse_metrics)], + % refresh stats 'R' times + ChanTable = read_table_rpc(Config, channel_created), + [[Pid ! emit_stats || {Pid, _, _} <- ChanTable ] || _ <- lists:seq(1, R)], + force_metric_gc(Config), + TableAfter = [ Pid || {Pid, _, _} <- read_table_rpc(Config, queue_metrics)], + TableAfter2 = [ Pid || {Pid, _, _} <- read_table_rpc(Config, queue_coarse_metrics)], + [ delete_queue(Chan, Q) || Q <- Queues], + rabbit_ct_client_helpers:close_connection(Conn), + (Table2 == TableAfter2) and (Table == TableAfter) and + (N == length(Table)) and (N == length(TableAfter)). + +connection_metric_count(Config, Ops) -> + add_rem_counter(Config, Ops, + {fun rabbit_ct_client_helpers:open_unmanaged_connection/1, + fun(Cfg) -> + rabbit_ct_client_helpers:close_connection(Cfg) + end}, + [ connection_created, + connection_metrics, + connection_coarse_metrics ]). + +channel_metric_count(Config, Ops) -> + Conn = rabbit_ct_client_helpers:open_unmanaged_connection(Config), + Result = add_rem_counter(Config, Ops, + {fun (_Config) -> + {ok, Chan} = amqp_connection:open_channel(Conn), + Chan + end, + fun amqp_channel:close/1}, + [ channel_created, + channel_metrics, + channel_process_metrics ]), + ok = rabbit_ct_client_helpers:close_connection(Conn), + Result. + +queue_metric_count(Config, Ops) -> + Conn = rabbit_ct_client_helpers:open_unmanaged_connection(Config), + {ok, Chan} = amqp_connection:open_channel(Conn), + AddFun = fun (_) -> + Queue = declare_queue(Chan), + ensure_exchange_metrics_populated(Chan, Queue), + ensure_channel_queue_metrics_populated(Chan, Queue), + force_channel_stats(Config), + Queue + end, + Result = add_rem_counter(Config, Ops, + {AddFun, + fun (Q) -> delete_queue(Chan, Q), + force_metric_gc(Config) + end}, [channel_queue_metrics, + channel_queue_exchange_metrics ]), + ok = rabbit_ct_client_helpers:close_connection(Conn), + Result. + +queue_metric_count_channel_per_queue(Config, Ops) -> + Conn = rabbit_ct_client_helpers:open_unmanaged_connection(Config), + AddFun = fun (_) -> + {ok, Chan} = amqp_connection:open_channel(Conn), + Queue = declare_queue(Chan), + ensure_exchange_metrics_populated(Chan, Queue), + ensure_channel_queue_metrics_populated(Chan, Queue), + force_channel_stats(Config), + {Chan, Queue} + end, + Result = add_rem_counter(Config, Ops, + {AddFun, + fun ({Chan, Q}) -> + delete_queue(Chan, Q), + force_metric_gc(Config) + end}, + [ channel_queue_metrics, + channel_queue_exchange_metrics ]), + ok = rabbit_ct_client_helpers:close_connection(Conn), + Result. + +add_rem_counter(Config, {Initial, Ops}, {AddFun, RemFun}, Tables) -> + Things = [ AddFun(Config) || _ <- lists:seq(1, Initial) ], + % either add or remove some things + {FinalLen, Things1} = + lists:foldl(fun(add, {L, Items}) -> + {L+1, [AddFun(Config) | Items]}; + (remove, {L, [H|Tail]}) -> + RemFun(H), + {L-1, Tail}; + (_, S) -> S end, + {Initial, Things}, + Ops), + force_metric_gc(Config), + TabLens = lists:map(fun(T) -> + length(read_table_rpc(Config, T)) + end, Tables), + [RemFun(Thing) || Thing <- Things1], + [FinalLen] == lists:usort(TabLens). + + +connection(Config) -> + Conn = rabbit_ct_client_helpers:open_unmanaged_connection(Config), + [_] = read_table_rpc(Config, connection_created), + [_] = read_table_rpc(Config, connection_metrics), + [_] = read_table_rpc(Config, connection_coarse_metrics), + ok = rabbit_ct_client_helpers:close_connection(Conn), + force_metric_gc(Config), + [] = read_table_rpc(Config, connection_created), + [] = read_table_rpc(Config, connection_metrics), + [] = read_table_rpc(Config, connection_coarse_metrics), + ok. + +channel(Config) -> + Conn = rabbit_ct_client_helpers:open_unmanaged_connection(Config), + {ok, Chan} = amqp_connection:open_channel(Conn), + [_] = read_table_rpc(Config, channel_created), + [_] = read_table_rpc(Config, channel_metrics), + [_] = read_table_rpc(Config, channel_process_metrics), + ok = amqp_channel:close(Chan), + [] = read_table_rpc(Config, channel_created), + [] = read_table_rpc(Config, channel_metrics), + [] = read_table_rpc(Config, channel_process_metrics), + ok = rabbit_ct_client_helpers:close_connection(Conn). + +channel_connection_close(Config) -> + Conn = rabbit_ct_client_helpers:open_unmanaged_connection(Config), + {ok, _} = amqp_connection:open_channel(Conn), + [_] = read_table_rpc(Config, channel_created), + [_] = read_table_rpc(Config, channel_metrics), + [_] = read_table_rpc(Config, channel_process_metrics), + ok = rabbit_ct_client_helpers:close_connection(Conn), + [] = read_table_rpc(Config, channel_created), + [] = read_table_rpc(Config, channel_metrics), + [] = read_table_rpc(Config, channel_process_metrics). + +channel_queue_delete_queue(Config) -> + Conn = rabbit_ct_client_helpers:open_unmanaged_connection(Config), + {ok, Chan} = amqp_connection:open_channel(Conn), + Queue = declare_queue(Chan), + ensure_exchange_metrics_populated(Chan, Queue), + ensure_channel_queue_metrics_populated(Chan, Queue), + force_channel_stats(Config), + [_] = read_table_rpc(Config, channel_queue_metrics), + [_] = read_table_rpc(Config, channel_queue_exchange_metrics), + + delete_queue(Chan, Queue), + force_metric_gc(Config), + % ensure removal of queue cleans up channel_queue metrics + [] = read_table_rpc(Config, channel_queue_exchange_metrics), + [] = read_table_rpc(Config, channel_queue_metrics), + ok = rabbit_ct_client_helpers:close_connection(Conn), + ok. + +channel_queue_exchange_consumer_close_connection(Config) -> + Conn = rabbit_ct_client_helpers:open_unmanaged_connection(Config), + {ok, Chan} = amqp_connection:open_channel(Conn), + Queue = declare_queue(Chan), + ensure_exchange_metrics_populated(Chan, Queue), + force_channel_stats(Config), + + [_] = read_table_rpc(Config, channel_exchange_metrics), + [_] = read_table_rpc(Config, channel_queue_exchange_metrics), + + ensure_channel_queue_metrics_populated(Chan, Queue), + force_channel_stats(Config), + [_] = read_table_rpc(Config, channel_queue_metrics), + + Sub = #'basic.consume'{queue = Queue}, + #'basic.consume_ok'{consumer_tag = _} = + amqp_channel:call(Chan, Sub), + + [_] = read_table_rpc(Config, consumer_created), + + ok = rabbit_ct_client_helpers:close_connection(Conn), + % ensure cleanup happened + force_metric_gc(Config), + [] = read_table_rpc(Config, channel_exchange_metrics), + [] = read_table_rpc(Config, channel_queue_exchange_metrics), + [] = read_table_rpc(Config, channel_queue_metrics), + [] = read_table_rpc(Config, consumer_created), + ok. + + + +%% ------------------------------------------------------------------- +%% Utilities +%% ------------------------------------------------------------------- + +declare_queue(Chan) -> + Declare = #'queue.declare'{durable = false, auto_delete = true}, + #'queue.declare_ok'{queue = Name} = amqp_channel:call(Chan, Declare), + Name. + +delete_queue(Chan, Name) -> + Delete = #'queue.delete'{queue = Name}, + #'queue.delete_ok'{} = amqp_channel:call(Chan, Delete). + +ensure_exchange_metrics_populated(Chan, RoutingKey) -> + % need to publish for exchange metrics to be populated + Publish = #'basic.publish'{routing_key = RoutingKey}, + amqp_channel:call(Chan, Publish, #amqp_msg{payload = <<"hello">>}). + +ensure_channel_queue_metrics_populated(Chan, Queue) -> + % need to get and wait for timer for channel queue metrics to be populated + Get = #'basic.get'{queue = Queue, no_ack=true}, + {#'basic.get_ok'{}, #amqp_msg{}} = amqp_channel:call(Chan, Get). + +force_channel_stats(Config) -> + [ Pid ! emit_stats || {Pid, _} <- read_table_rpc(Config, channel_created) ], + timer:sleep(100). + +read_table_rpc(Config, Table) -> + rabbit_ct_broker_helpers:rpc(Config, 0, ?MODULE, read_table, [Table]). + +clean_core_metrics(Config) -> + [ rabbit_ct_broker_helpers:rpc(Config, 0, ets, delete_all_objects, [Table]) + || {Table, _} <- ?CORE_TABLES]. + +read_table(Table) -> + ets:tab2list(Table). + +force_metric_gc(Config) -> + timer:sleep(300), + rabbit_ct_broker_helpers:rpc(Config, 0, erlang, send, + [rabbit_core_metrics_gc, start_gc]), + rabbit_ct_broker_helpers:rpc(Config, 0, gen_server, call, + [rabbit_core_metrics_gc, test]). diff --git a/deps/rabbit/test/mirrored_supervisor_SUITE.erl b/deps/rabbit/test/mirrored_supervisor_SUITE.erl new file mode 100644 index 0000000000..7ce88cfdaa --- /dev/null +++ b/deps/rabbit/test/mirrored_supervisor_SUITE.erl @@ -0,0 +1,328 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2011-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(mirrored_supervisor_SUITE). + +-behaviour(mirrored_supervisor). + +-include_lib("common_test/include/ct.hrl"). + +-compile(export_all). + +-define(MS, mirrored_supervisor). +-define(SERVER, mirrored_supervisor_SUITE_gs). + +all() -> + [ + migrate, + migrate_twice, + already_there, + delete_restart, + which_children, + large_group, + childspecs_at_init, + anonymous_supervisors, + no_migration_on_shutdown, + start_idempotence, + unsupported, + ignore, + startup_failure + ]. + +init_per_suite(Config) -> + ok = application:set_env(mnesia, dir, ?config(priv_dir, Config)), + ok = application:start(mnesia), + lists:foreach( + fun ({Tab, TabDef}) -> + TabDef1 = proplists:delete(match, TabDef), + case mnesia:create_table(Tab, TabDef1) of + {atomic, ok} -> + ok; + {aborted, Reason} -> + throw({error, + {table_creation_failed, Tab, TabDef1, Reason}}) + end + end, mirrored_supervisor:table_definitions()), + Config. + +end_per_suite(Config) -> + ok = application:stop(mnesia), + Config. + +%% --------------------------------------------------------------------------- +%% Functional tests +%% --------------------------------------------------------------------------- + +%% Simplest test +migrate(_Config) -> + passed = with_sups( + fun([A, _]) -> + {ok, _} = ?MS:start_child(a, childspec(worker)), + Pid1 = pid_of(worker), + kill_registered(A, Pid1), + Pid2 = pid_of(worker), + false = (Pid1 =:= Pid2) + end, [a, b]). + +%% Is migration transitive? +migrate_twice(_Config) -> + passed = with_sups( + fun([A, B]) -> + {ok, _} = ?MS:start_child(a, childspec(worker)), + Pid1 = pid_of(worker), + kill_registered(A, Pid1), + {ok, C} = start_sup(c), + Pid2 = pid_of(worker), + kill_registered(B, Pid2), + Pid3 = pid_of(worker), + false = (Pid1 =:= Pid3), + kill(C) + end, [a, b]). + +%% Can't start the same child twice +already_there(_Config) -> + passed = with_sups( + fun([_, _]) -> + S = childspec(worker), + {ok, Pid} = ?MS:start_child(a, S), + {error, {already_started, Pid}} = ?MS:start_child(b, S) + end, [a, b]). + +%% Deleting and restarting should work as per a normal supervisor +delete_restart(_Config) -> + passed = with_sups( + fun([_, _]) -> + S = childspec(worker), + {ok, Pid1} = ?MS:start_child(a, S), + {error, running} = ?MS:delete_child(a, worker), + ok = ?MS:terminate_child(a, worker), + ok = ?MS:delete_child(a, worker), + {ok, Pid2} = ?MS:start_child(b, S), + false = (Pid1 =:= Pid2), + ok = ?MS:terminate_child(b, worker), + {ok, Pid3} = ?MS:restart_child(b, worker), + Pid3 = pid_of(worker), + false = (Pid2 =:= Pid3), + %% Not the same supervisor as the worker is on + ok = ?MS:terminate_child(a, worker), + ok = ?MS:delete_child(a, worker), + {ok, Pid4} = ?MS:start_child(a, S), + false = (Pid3 =:= Pid4) + end, [a, b]). + +which_children(_Config) -> + passed = with_sups( + fun([A, B] = Both) -> + ?MS:start_child(A, childspec(worker)), + assert_wc(Both, fun ([C]) -> true = is_pid(wc_pid(C)) end), + ok = ?MS:terminate_child(a, worker), + assert_wc(Both, fun ([C]) -> undefined = wc_pid(C) end), + {ok, _} = ?MS:restart_child(a, worker), + assert_wc(Both, fun ([C]) -> true = is_pid(wc_pid(C)) end), + ?MS:start_child(B, childspec(worker2)), + assert_wc(Both, fun (C) -> 2 = length(C) end) + end, [a, b]). + +assert_wc(Sups, Fun) -> + [Fun(?MS:which_children(Sup)) || Sup <- Sups]. + +wc_pid(Child) -> + {worker, Pid, worker, [?MODULE]} = Child, + Pid. + +%% Not all the members of the group should actually do the failover +large_group(_Config) -> + passed = with_sups( + fun([A, _, _, _]) -> + {ok, _} = ?MS:start_child(a, childspec(worker)), + Pid1 = pid_of(worker), + kill_registered(A, Pid1), + Pid2 = pid_of(worker), + false = (Pid1 =:= Pid2) + end, [a, b, c, d]). + +%% Do childspecs work when returned from init? +childspecs_at_init(_Config) -> + S = childspec(worker), + passed = with_sups( + fun([A, _]) -> + Pid1 = pid_of(worker), + kill_registered(A, Pid1), + Pid2 = pid_of(worker), + false = (Pid1 =:= Pid2) + end, [{a, [S]}, {b, [S]}]). + +anonymous_supervisors(_Config) -> + passed = with_sups( + fun([A, _B]) -> + {ok, _} = ?MS:start_child(A, childspec(worker)), + Pid1 = pid_of(worker), + kill_registered(A, Pid1), + Pid2 = pid_of(worker), + false = (Pid1 =:= Pid2) + end, [anon, anon]). + +%% When a mirrored_supervisor terminates, we should not migrate, but +%% the whole supervisor group should shut down. To test this we set up +%% a situation where the gen_server will only fail if it's running +%% under the supervisor called 'evil'. It should not migrate to +%% 'good' and survive, rather the whole group should go away. +no_migration_on_shutdown(_Config) -> + passed = with_sups( + fun([Evil, _]) -> + {ok, _} = ?MS:start_child(Evil, childspec(worker)), + try + call(worker, ping, 1000, 100), + exit(worker_should_not_have_migrated) + catch exit:{timeout_waiting_for_server, _, _} -> + ok + end + end, [evil, good]). + +start_idempotence(_Config) -> + passed = with_sups( + fun([_]) -> + CS = childspec(worker), + {ok, Pid} = ?MS:start_child(a, CS), + {error, {already_started, Pid}} = ?MS:start_child(a, CS), + ?MS:terminate_child(a, worker), + {error, already_present} = ?MS:start_child(a, CS) + end, [a]). + +unsupported(_Config) -> + try + ?MS:start_link({global, foo}, get_group(group), fun tx_fun/1, ?MODULE, + {one_for_one, []}), + exit(no_global) + catch error:badarg -> + ok + end, + try + {ok, _} = ?MS:start_link({local, foo}, get_group(group), + fun tx_fun/1, ?MODULE, {simple_one_for_one, []}), + exit(no_sofo) + catch error:badarg -> + ok + end. + +%% Just test we don't blow up +ignore(_Config) -> + ?MS:start_link({local, foo}, get_group(group), fun tx_fun/1, ?MODULE, + {fake_strategy_for_ignore, []}). + +startup_failure(_Config) -> + [test_startup_failure(F) || F <- [want_error, want_exit]]. + +test_startup_failure(Fail) -> + process_flag(trap_exit, true), + ?MS:start_link(get_group(group), fun tx_fun/1, ?MODULE, + {one_for_one, [childspec(Fail)]}), + receive + {'EXIT', _, shutdown} -> + ok + after 1000 -> + exit({did_not_exit, Fail}) + end, + process_flag(trap_exit, false). + +%% --------------------------------------------------------------------------- + +with_sups(Fun, Sups) -> + inc_group(), + Pids = [begin {ok, Pid} = start_sup(Sup), Pid end || Sup <- Sups], + Fun(Pids), + [kill(Pid) || Pid <- Pids, is_process_alive(Pid)], + timer:sleep(500), + passed. + +start_sup(Spec) -> + start_sup(Spec, group). + +start_sup({Name, ChildSpecs}, Group) -> + {ok, Pid} = start_sup0(Name, get_group(Group), ChildSpecs), + %% We are not a supervisor, when we kill the supervisor we do not + %% want to die! + unlink(Pid), + {ok, Pid}; + +start_sup(Name, Group) -> + start_sup({Name, []}, Group). + +start_sup0(anon, Group, ChildSpecs) -> + ?MS:start_link(Group, fun tx_fun/1, ?MODULE, + {one_for_one, ChildSpecs}); + +start_sup0(Name, Group, ChildSpecs) -> + ?MS:start_link({local, Name}, Group, fun tx_fun/1, ?MODULE, + {one_for_one, ChildSpecs}). + +childspec(Id) -> + {Id,{?SERVER, start_link, [Id]}, transient, 16#ffffffff, worker, [?MODULE]}. + +pid_of(Id) -> + {received, Pid, ping} = call(Id, ping), + Pid. + +tx_fun(Fun) -> + case mnesia:sync_transaction(Fun) of + {atomic, Result} -> Result; + {aborted, Reason} -> throw({error, Reason}) + end. + +inc_group() -> + Count = case get(counter) of + undefined -> 0; + C -> C + end + 1, + put(counter, Count). + +get_group(Group) -> + {Group, get(counter)}. + +call(Id, Msg) -> call(Id, Msg, 10*1000, 100). + +call(Id, Msg, MaxDelay, Decr) -> + call(Id, Msg, MaxDelay, Decr, undefined). + +call(Id, Msg, 0, _Decr, Stacktrace) -> + exit({timeout_waiting_for_server, {Id, Msg}, Stacktrace}); + +call(Id, Msg, MaxDelay, Decr, _) -> + try + gen_server:call(Id, Msg, infinity) + catch exit:_:Stacktrace -> timer:sleep(Decr), + call(Id, Msg, MaxDelay - Decr, Decr, Stacktrace) + end. + +kill(Pid) -> kill(Pid, []). +kill(Pid, Wait) when is_pid(Wait) -> kill(Pid, [Wait]); +kill(Pid, Waits) -> + erlang:monitor(process, Pid), + [erlang:monitor(process, P) || P <- Waits], + exit(Pid, bang), + kill_wait(Pid), + [kill_wait(P) || P <- Waits]. + +kill_registered(Pid, Child) -> + {registered_name, Name} = erlang:process_info(Child, registered_name), + kill(Pid, Child), + false = (Child =:= whereis(Name)), + ok. + +kill_wait(Pid) -> + receive + {'DOWN', _Ref, process, Pid, _Reason} -> + ok + end. + +%% --------------------------------------------------------------------------- + +init({fake_strategy_for_ignore, _ChildSpecs}) -> + ignore; + +init({Strategy, ChildSpecs}) -> + {ok, {{Strategy, 0, 1}, ChildSpecs}}. diff --git a/deps/rabbit/test/mirrored_supervisor_SUITE_gs.erl b/deps/rabbit/test/mirrored_supervisor_SUITE_gs.erl new file mode 100644 index 0000000000..62245231d7 --- /dev/null +++ b/deps/rabbit/test/mirrored_supervisor_SUITE_gs.erl @@ -0,0 +1,57 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2011-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(mirrored_supervisor_SUITE_gs). + +%% Dumb gen_server we can supervise + +-export([start_link/1]). + +-export([init/1, handle_call/3, handle_info/2, terminate/2, code_change/3, + handle_cast/2]). + +-behaviour(gen_server). + +-define(MS, mirrored_supervisor). + +start_link(want_error) -> + {error, foo}; + +start_link(want_exit) -> + exit(foo); + +start_link(Id) -> + gen_server:start_link({local, Id}, ?MODULE, [], []). + +%% --------------------------------------------------------------------------- + +init([]) -> + {ok, state}. + +handle_call(Msg, _From, State) -> + die_if_my_supervisor_is_evil(), + {reply, {received, self(), Msg}, State}. + +handle_cast(_Msg, State) -> + {noreply, State}. + +handle_info(_Info, State) -> + {noreply, State}. + +terminate(_Reason, _State) -> + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +die_if_my_supervisor_is_evil() -> + try lists:keysearch(self(), 2, ?MS:which_children(evil)) of + false -> ok; + _ -> exit(doooom) + catch + exit:{noproc, _} -> ok + end. diff --git a/deps/rabbit/test/msg_store_SUITE.erl b/deps/rabbit/test/msg_store_SUITE.erl new file mode 100644 index 0000000000..e349aa4443 --- /dev/null +++ b/deps/rabbit/test/msg_store_SUITE.erl @@ -0,0 +1,53 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2011-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(msg_store_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("rabbit_common/include/rabbit.hrl"). + +-compile(export_all). + +-define(T(Fun, Args), (catch apply(rabbit, Fun, Args))). + +all() -> + [ + parameter_validation + ]. + +parameter_validation(_Config) -> + %% make sure it works with default values + ok = ?T(validate_msg_store_io_batch_size_and_credit_disc_bound, + [?CREDIT_DISC_BOUND, ?IO_BATCH_SIZE]), + + %% IO_BATCH_SIZE must be greater than CREDIT_DISC_BOUND initial credit + ok = ?T(validate_msg_store_io_batch_size_and_credit_disc_bound, + [{4000, 800}, 5000]), + {error, _} = ?T(validate_msg_store_io_batch_size_and_credit_disc_bound, + [{4000, 800}, 1500]), + + %% All values must be integers + {error, _} = ?T(validate_msg_store_io_batch_size_and_credit_disc_bound, + [{2000, 500}, "1500"]), + {error, _} = ?T(validate_msg_store_io_batch_size_and_credit_disc_bound, + [{"2000", 500}, abc]), + {error, _} = ?T(validate_msg_store_io_batch_size_and_credit_disc_bound, + [{2000, "500"}, 2048]), + + %% CREDIT_DISC_BOUND must be a tuple + {error, _} = ?T(validate_msg_store_io_batch_size_and_credit_disc_bound, + [[2000, 500], 1500]), + {error, _} = ?T(validate_msg_store_io_batch_size_and_credit_disc_bound, + [2000, 1500]), + + %% config values can't be smaller than default values + {error, _} = ?T(validate_msg_store_io_batch_size_and_credit_disc_bound, + [{1999, 500}, 2048]), + {error, _} = ?T(validate_msg_store_io_batch_size_and_credit_disc_bound, + [{2000, 499}, 2048]), + {error, _} = ?T(validate_msg_store_io_batch_size_and_credit_disc_bound, + [{2000, 500}, 2047]). diff --git a/deps/rabbit/test/peer_discovery_classic_config_SUITE.erl b/deps/rabbit/test/peer_discovery_classic_config_SUITE.erl new file mode 100644 index 0000000000..ddb753adf8 --- /dev/null +++ b/deps/rabbit/test/peer_discovery_classic_config_SUITE.erl @@ -0,0 +1,179 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2011-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(peer_discovery_classic_config_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-import(rabbit_ct_broker_helpers, [ + cluster_members_online/2 +]). + +-compile(export_all). + +all() -> + [ + {group, non_parallel} + ]. + +groups() -> + [ + {non_parallel, [], [ + successful_discovery, + successful_discovery_with_a_subset_of_nodes_coming_online, + no_nodes_configured + ]} + ]. + +suite() -> + [ + {timetrap, {minutes, 5}} + ]. + + +%% +%% Setup/teardown. +%% + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(_, Config) -> + Config. + +end_per_group(_, Config) -> + Config. + +init_per_testcase(successful_discovery = Testcase, Config) -> + Config1 = rabbit_ct_helpers:testcase_started(Config, Testcase), + + N = 3, + NodeNames = [ + list_to_atom(rabbit_misc:format("~s-~b", [Testcase, I])) + || I <- lists:seq(1, N) + ], + Config2 = rabbit_ct_helpers:set_config(Config1, [ + {rmq_nodename_suffix, Testcase}, + %% note: this must not include the host part + {rmq_nodes_count, NodeNames}, + {rmq_nodes_clustered, false} + ]), + NodeNamesWithHostname = [rabbit_nodes:make({Name, "localhost"}) || Name <- NodeNames], + Config3 = rabbit_ct_helpers:merge_app_env(Config2, + {rabbit, [ + {cluster_nodes, {NodeNamesWithHostname, disc}}, + {cluster_formation, [ + {randomized_startup_delay_range, {1, 10}} + ]} + ]}), + rabbit_ct_helpers:run_steps(Config3, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()); +init_per_testcase(successful_discovery_with_a_subset_of_nodes_coming_online = Testcase, Config) -> + Config1 = rabbit_ct_helpers:testcase_started(Config, Testcase), + + N = 2, + NodeNames = [ + list_to_atom(rabbit_misc:format("~s-~b", [Testcase, I])) + || I <- lists:seq(1, N) + ], + Config2 = rabbit_ct_helpers:set_config(Config1, [ + {rmq_nodename_suffix, Testcase}, + %% note: this must not include the host part + {rmq_nodes_count, NodeNames}, + {rmq_nodes_clustered, false} + ]), + NodeNamesWithHostname = [rabbit_nodes:make({Name, "localhost"}) || Name <- [nonexistent | NodeNames]], + %% reduce retry time since we know one node on the list does + %% not exist and not just unreachable + Config3 = rabbit_ct_helpers:merge_app_env(Config2, + {rabbit, [ + {cluster_formation, [ + {discovery_retry_limit, 10}, + {discovery_retry_interval, 200} + ]}, + {cluster_nodes, {NodeNamesWithHostname, disc}}, + {cluster_formation, [ + {randomized_startup_delay_range, {1, 10}} + ]} + ]}), + rabbit_ct_helpers:run_steps(Config3, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()); +init_per_testcase(no_nodes_configured = Testcase, Config) -> + Config1 = rabbit_ct_helpers:testcase_started(Config, Testcase), + Config2 = rabbit_ct_helpers:set_config(Config1, [ + {rmq_nodename_suffix, Testcase}, + {rmq_nodes_count, 2}, + {rmq_nodes_clustered, false} + ]), + Config3 = rabbit_ct_helpers:merge_app_env(Config2, + {rabbit, [ + {cluster_nodes, {[], disc}}, + {cluster_formation, [ + {randomized_startup_delay_range, {1, 10}} + ]} + ]}), + rabbit_ct_helpers:run_steps(Config3, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()); +init_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_started(Config, Testcase), + rabbit_ct_helpers:run_steps(Config, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()). + + +end_per_testcase(Testcase, Config) -> + Config1 = rabbit_ct_helpers:run_steps(Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()), + rabbit_ct_helpers:testcase_finished(Config1, Testcase). + + +%% +%% Test cases +%% +successful_discovery(Config) -> + Condition = fun() -> + 3 =:= length(cluster_members_online(Config, 0)) andalso + 3 =:= length(cluster_members_online(Config, 1)) + end, + await_cluster(Config, Condition, [1, 2]). + +successful_discovery_with_a_subset_of_nodes_coming_online(Config) -> + Condition = fun() -> + 2 =:= length(cluster_members_online(Config, 0)) andalso + 2 =:= length(cluster_members_online(Config, 1)) + end, + await_cluster(Config, Condition, [1]). + +no_nodes_configured(Config) -> + Condition = fun() -> length(cluster_members_online(Config, 0)) < 2 end, + await_cluster(Config, Condition, [1]). + +reset_and_restart_node(Config, I) when is_integer(I) andalso I >= 0 -> + Name = rabbit_ct_broker_helpers:get_node_config(Config, I, nodename), + rabbit_control_helper:command(stop_app, Name), + rabbit_ct_broker_helpers:reset_node(Config, Name), + rabbit_control_helper:command(start_app, Name). + +await_cluster(Config, Condition, Nodes) -> + try + rabbit_ct_helpers:await_condition(Condition, 30000) + catch + exit:{test_case_failed, _} -> + ct:pal(?LOW_IMPORTANCE, "Possible dead-lock; resetting/restarting these nodes: ~p", [Nodes]), + [reset_and_restart_node(Config, N) || N <- Nodes], + rabbit_ct_helpers:await_condition(Condition, 30000) + end. diff --git a/deps/rabbit/test/peer_discovery_dns_SUITE.erl b/deps/rabbit/test/peer_discovery_dns_SUITE.erl new file mode 100644 index 0000000000..5184bc11eb --- /dev/null +++ b/deps/rabbit/test/peer_discovery_dns_SUITE.erl @@ -0,0 +1,104 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2011-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(peer_discovery_dns_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-compile(export_all). + +all() -> + [ + {group, non_parallel} + ]. + +groups() -> + [ + {non_parallel, [], [ + hostname_discovery_with_long_node_names, + hostname_discovery_with_short_node_names, + node_discovery_with_long_node_names, + node_discovery_with_short_node_names, + test_aaaa_record_hostname_discovery + ]} + ]. + +suite() -> + [ + %% If a test hangs, no need to wait for 30 minutes. + {timetrap, {minutes, 1}} + ]. + + +%% ------------------------------------------------------------------- +%% Testsuite setup/teardown. +%% ------------------------------------------------------------------- + +%% These are stable, publicly resolvable hostnames that +%% both return A and AAAA records that reverse resolve. +-define(DISCOVERY_ENDPOINT_RECORD_A, "dns.google"). +-define(DISCOVERY_ENDPOINT_RECORD_AAAA, "dns.google"). + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + + +init_per_testcase(test_aaaa_record, Config) -> + case inet_res:lookup(?DISCOVERY_ENDPOINT_RECORD_AAAA, in, aaaa) of + [] -> + {skip, "pre-configured AAAA record does not resolve, skipping"}; + [_ | _] -> + Config + end; + +init_per_testcase(_Testcase, Config) -> + case inet_res:lookup(?DISCOVERY_ENDPOINT_RECORD_A, in, a) of + [] -> + {skip, "pre-configured *.rabbitmq.com record does not resolve, skipping"}; + [_ | _] -> + Config + end. + + +end_per_testcase(_Testcase, Config) -> + case inet_res:lookup(?DISCOVERY_ENDPOINT_RECORD_A, in, a) of + [] -> + {skip, "pre-configured *.rabbitmq.com record does not resolve, skipping"}; + [_ | _] -> + Config + end. + + +%% ------------------------------------------------------------------- +%% Test cases +%% ------------------------------------------------------------------- + +test_aaaa_record_hostname_discovery(_) -> + Result = rabbit_peer_discovery_dns:discover_hostnames(?DISCOVERY_ENDPOINT_RECORD_AAAA, true), + ?assert(string:str(lists:flatten(Result), "dns.google") > 0). + +hostname_discovery_with_long_node_names(_) -> + Result = rabbit_peer_discovery_dns:discover_hostnames(?DISCOVERY_ENDPOINT_RECORD_A, true), + ?assert(lists:member("dns.google", Result)). + +hostname_discovery_with_short_node_names(_) -> + Result = rabbit_peer_discovery_dns:discover_hostnames(?DISCOVERY_ENDPOINT_RECORD_A, false), + ?assert(lists:member("dns", Result)). + +node_discovery_with_long_node_names(_) -> + Result = rabbit_peer_discovery_dns:discover_nodes(?DISCOVERY_ENDPOINT_RECORD_A, true), + ?assert(lists:member('ct_rabbit@dns.google', Result)). + +node_discovery_with_short_node_names(_) -> + Result = rabbit_peer_discovery_dns:discover_nodes(?DISCOVERY_ENDPOINT_RECORD_A, false), + ?assert(lists:member(ct_rabbit@dns, Result)). diff --git a/deps/rabbit/test/per_user_connection_channel_limit_SUITE.erl b/deps/rabbit/test/per_user_connection_channel_limit_SUITE.erl new file mode 100644 index 0000000000..43c860c8bd --- /dev/null +++ b/deps/rabbit/test/per_user_connection_channel_limit_SUITE.erl @@ -0,0 +1,1651 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(per_user_connection_channel_limit_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("rabbitmq_ct_helpers/include/rabbit_assert.hrl"). + +-compile(export_all). + +all() -> + [ + {group, cluster_size_1_network}, + {group, cluster_size_2_network}, + {group, cluster_size_2_direct} + ]. + +groups() -> + ClusterSize1Tests = [ + most_basic_single_node_connection_and_channel_count, + single_node_single_user_connection_and_channel_count, + single_node_multiple_users_connection_and_channel_count, + single_node_list_in_user, + single_node_single_user_limit, + single_node_single_user_zero_limit, + single_node_single_user_clear_limits, + single_node_multiple_users_clear_limits, + single_node_multiple_users_limit, + single_node_multiple_users_zero_limit + + ], + ClusterSize2Tests = [ + most_basic_cluster_connection_and_channel_count, + cluster_single_user_connection_and_channel_count, + cluster_multiple_users_connection_and_channel_count, + cluster_node_restart_connection_and_channel_count, + cluster_node_list_on_node, + cluster_single_user_limit, + cluster_single_user_limit2, + cluster_single_user_zero_limit, + cluster_single_user_clear_limits, + cluster_multiple_users_clear_limits, + cluster_multiple_users_zero_limit + ], + [ + {cluster_size_1_network, [], ClusterSize1Tests}, + {cluster_size_2_network, [], ClusterSize2Tests}, + {cluster_size_2_direct, [], ClusterSize2Tests} + ]. + +suite() -> + [ + %% If a test hangs, no need to wait for 30 minutes. + {timetrap, {minutes, 8}} + ]. + +%% ------------------------------------------------------------------- +%% Testsuite setup/teardown. +%% ------------------------------------------------------------------- + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(cluster_size_1_network, Config) -> + Config1 = rabbit_ct_helpers:set_config(Config, [{connection_type, network}]), + init_per_multinode_group(cluster_size_1_network, Config1, 1); +init_per_group(cluster_size_2_network, Config) -> + Config1 = rabbit_ct_helpers:set_config(Config, [{connection_type, network}]), + init_per_multinode_group(cluster_size_2_network, Config1, 2); +init_per_group(cluster_size_2_direct, Config) -> + Config1 = rabbit_ct_helpers:set_config(Config, [{connection_type, direct}]), + init_per_multinode_group(cluster_size_2_direct, Config1, 2); + +init_per_group(cluster_rename, Config) -> + init_per_multinode_group(cluster_rename, Config, 2). + +init_per_multinode_group(Group, Config, NodeCount) -> + Suffix = rabbit_ct_helpers:testcase_absname(Config, "", "-"), + Config1 = rabbit_ct_helpers:set_config(Config, [ + {rmq_nodes_count, NodeCount}, + {rmq_nodename_suffix, Suffix} + ]), + case Group of + cluster_rename -> + % The broker is managed by {init,end}_per_testcase(). + Config1; + _ -> + Config2 = rabbit_ct_helpers:run_steps( + Config1, rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()), + EnableFF = rabbit_ct_broker_helpers:enable_feature_flag( + Config2, user_limits), + case EnableFF of + ok -> + Config2; + Skip -> + end_per_group(Group, Config2), + Skip + end + end. + +end_per_group(cluster_rename, Config) -> + % The broker is managed by {init,end}_per_testcase(). + Config; +end_per_group(_Group, Config) -> + rabbit_ct_helpers:run_steps(Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()). + +init_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_started(Config, Testcase), + clear_all_connection_tracking_tables(Config), + clear_all_channel_tracking_tables(Config), + Config. + +end_per_testcase(Testcase, Config) -> + clear_all_connection_tracking_tables(Config), + clear_all_channel_tracking_tables(Config), + rabbit_ct_helpers:testcase_finished(Config, Testcase). + +clear_all_connection_tracking_tables(Config) -> + [rabbit_ct_broker_helpers:rpc(Config, + N, + rabbit_connection_tracking, + clear_tracking_tables, + []) || N <- rabbit_ct_broker_helpers:get_node_configs(Config, nodename)]. + +clear_all_channel_tracking_tables(Config) -> + [rabbit_ct_broker_helpers:rpc(Config, + N, + rabbit_channel_tracking, + clear_tracking_tables, + []) || N <- rabbit_ct_broker_helpers:get_node_configs(Config, nodename)]. + +%% ------------------------------------------------------------------- +%% Test cases. +%% ------------------------------------------------------------------- + +most_basic_single_node_connection_and_channel_count(Config) -> + Username = proplists:get_value(rmq_username, Config), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 0 andalso + count_channels_of_user(Config, Username) =:= 0 + end), + + [Conn] = open_connections(Config, [0]), + [Chan] = open_channels(Conn, 1), + + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 1 andalso + count_channels_of_user(Config, Username) =:= 1 + end), + close_channels([Chan]), + rabbit_ct_helpers:await_condition( + fun () -> + count_channels_of_user(Config, Username) =:= 0 + end), + close_connections([Conn]), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 0 + end). + +single_node_single_user_connection_and_channel_count(Config) -> + Username = proplists:get_value(rmq_username, Config), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 0 andalso + count_channels_of_user(Config, Username) =:= 0 + end), + + [Conn1] = open_connections(Config, [0]), + [Chan1] = open_channels(Conn1, 1), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 1 andalso + count_channels_of_user(Config, Username) =:= 1 + end), + close_channels([Chan1]), + rabbit_ct_helpers:await_condition( + fun () -> + count_channels_of_user(Config, Username) =:= 0 + end), + close_connections([Conn1]), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 0 + end), + + [Conn2] = open_connections(Config, [0]), + Chans2 = [_|_] = open_channels(Conn2, 5), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 1 andalso + count_channels_of_user(Config, Username) =:= 5 + end), + + [Conn3] = open_connections(Config, [0]), + Chans3 = [_|_] = open_channels(Conn3, 5), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 2 andalso + count_channels_of_user(Config, Username) =:= 10 + end), + + [Conn4] = open_connections(Config, [0]), + _Chans4 = [_|_] = open_channels(Conn4, 5), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 3 andalso + count_channels_of_user(Config, Username) =:= 15 + end), + + close_connections([Conn4]), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 2 andalso + count_channels_of_user(Config, Username) =:= 10 + end), + + [Conn5] = open_connections(Config, [0]), + Chans5 = [_|_] = open_channels(Conn5, 5), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 3 andalso + count_channels_of_user(Config, Username) =:= 15 + end), + + close_channels(Chans2 ++ Chans3 ++ Chans5), + rabbit_ct_helpers:await_condition( + fun () -> + count_channels_of_user(Config, Username) =:= 0 + end), + + close_connections([Conn2, Conn3, Conn5]), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 0 + end). + +single_node_multiple_users_connection_and_channel_count(Config) -> + Username1 = <<"guest1">>, + Username2 = <<"guest2">>, + + set_up_user(Config, Username1), + set_up_user(Config, Username2), + + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username1) =:= 0 andalso + count_channels_of_user(Config, Username1) =:= 0 + end), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username2) =:= 0 andalso + count_channels_of_user(Config, Username2) =:= 0 + end), + + [Conn1] = open_connections(Config, [{0, Username1}]), + Chans1 = [_|_] = open_channels(Conn1, 5), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username1) =:= 1 andalso + count_channels_of_user(Config, Username1) =:= 5 + end), + close_channels(Chans1), + rabbit_ct_helpers:await_condition( + fun () -> + count_channels_of_user(Config, Username1) =:= 0 + end), + ?assertEqual(0, count_channels_of_user(Config, Username1)), + close_connections([Conn1]), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username1) =:= 0 andalso + count_channels_of_user(Config, Username1) =:= 0 + end), + + [Conn2] = open_connections(Config, [{0, Username2}]), + Chans2 = [_|_] = open_channels(Conn2, 5), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username2) =:= 1 andalso + count_channels_of_user(Config, Username2) =:= 5 + end), + + [Conn3] = open_connections(Config, [{0, Username1}]), + Chans3 = [_|_] = open_channels(Conn3, 5), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username1) =:= 1 andalso + count_channels_of_user(Config, Username1) =:= 5 + end), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username2) =:= 1 andalso + count_channels_of_user(Config, Username2) =:= 5 + end), + + [Conn4] = open_connections(Config, [{0, Username1}]), + _Chans4 = [_|_] = open_channels(Conn4, 5), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username1) =:= 2 andalso + count_channels_of_user(Config, Username1) =:= 10 + end), + + close_connections([Conn4]), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username1) =:= 1 andalso + count_channels_of_user(Config, Username1) =:= 5 + end), + + [Conn5] = open_connections(Config, [{0, Username2}]), + Chans5 = [_|_] = open_channels(Conn5, 5), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username2) =:= 2 andalso + count_channels_of_user(Config, Username2) =:= 10 + end), + + [Conn6] = open_connections(Config, [{0, Username2}]), + Chans6 = [_|_] = open_channels(Conn6, 5), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username2) =:= 3 andalso + count_channels_of_user(Config, Username2) =:= 15 + end), + + close_channels(Chans2 ++ Chans3 ++ Chans5 ++ Chans6), + rabbit_ct_helpers:await_condition( + fun () -> + count_channels_of_user(Config, Username1) =:= 0 andalso + count_channels_of_user(Config, Username2) =:= 0 + end), + + close_connections([Conn2, Conn3, Conn5, Conn6]), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username1) =:= 0 andalso + count_connections_of_user(Config, Username2) =:= 0 + end), + + rabbit_ct_broker_helpers:delete_user(Config, Username1), + rabbit_ct_broker_helpers:delete_user(Config, Username2). + +single_node_list_in_user(Config) -> + Username1 = <<"guest1">>, + Username2 = <<"guest2">>, + + set_up_user(Config, Username1), + set_up_user(Config, Username2), + + rabbit_ct_helpers:await_condition( + fun () -> + length(connections_in(Config, Username1)) =:= 0 andalso + length(connections_in(Config, Username2)) =:= 0 + end), + + ?assertEqual(0, length(channels_in(Config, Username1))), + ?assertEqual(0, length(channels_in(Config, Username2))), + + [Conn1] = open_connections(Config, [{0, Username1}]), + [Chan1] = open_channels(Conn1, 1), + [#tracked_connection{username = Username1}] = connections_in(Config, Username1), + [#tracked_channel{username = Username1}] = channels_in(Config, Username1), + close_channels([Chan1]), + rabbit_ct_helpers:await_condition( + fun () -> + length(channels_in(Config, Username1)) =:= 0 + end), + close_connections([Conn1]), + rabbit_ct_helpers:await_condition( + fun () -> + length(connections_in(Config, Username1)) =:= 0 + end), + + [Conn2] = open_connections(Config, [{0, Username2}]), + [Chan2] = open_channels(Conn2, 1), + [#tracked_connection{username = Username2}] = connections_in(Config, Username2), + [#tracked_channel{username = Username2}] = channels_in(Config, Username2), + + [Conn3] = open_connections(Config, [{0, Username1}]), + [Chan3] = open_channels(Conn3, 1), + [#tracked_connection{username = Username1}] = connections_in(Config, Username1), + [#tracked_channel{username = Username1}] = channels_in(Config, Username1), + + [Conn4] = open_connections(Config, [{0, Username1}]), + [_Chan4] = open_channels(Conn4, 1), + close_connections([Conn4]), + [#tracked_connection{username = Username1}] = connections_in(Config, Username1), + [#tracked_channel{username = Username1}] = channels_in(Config, Username1), + + [Conn5, Conn6] = open_connections(Config, [{0, Username2}, {0, Username2}]), + [Chan5] = open_channels(Conn5, 1), + [Chan6] = open_channels(Conn6, 1), + [<<"guest1">>, <<"guest2">>] = + lists:usort(lists:map(fun (#tracked_connection{username = V}) -> V end, + all_connections(Config))), + [<<"guest1">>, <<"guest2">>] = + lists:usort(lists:map(fun (#tracked_channel{username = V}) -> V end, + all_channels(Config))), + + close_channels([Chan2, Chan3, Chan5, Chan6]), + rabbit_ct_helpers:await_condition( + fun () -> + length(all_channels(Config)) =:= 0 + end), + + close_connections([Conn2, Conn3, Conn5, Conn6]), + rabbit_ct_helpers:await_condition( + fun () -> + length(all_connections(Config)) =:= 0 + end), + + rabbit_ct_broker_helpers:delete_user(Config, Username1), + rabbit_ct_broker_helpers:delete_user(Config, Username2). + +most_basic_cluster_connection_and_channel_count(Config) -> + Username = proplists:get_value(rmq_username, Config), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 0 andalso + count_channels_of_user(Config, Username) =:= 0 + end), + + [Conn1] = open_connections(Config, [0]), + Chans1 = [_|_] = open_channels(Conn1, 5), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 1 andalso + count_channels_of_user(Config, Username) =:= 5 + end), + ?assertEqual(1, count_connections_of_user(Config, Username)), + ?assertEqual(5, count_channels_of_user(Config, Username)), + + [Conn2] = open_connections(Config, [1]), + Chans2 = [_|_] = open_channels(Conn2, 5), + ?assertEqual(2, count_connections_of_user(Config, Username)), + ?assertEqual(10, count_channels_of_user(Config, Username)), + + [Conn3] = open_connections(Config, [1]), + Chans3 = [_|_] = open_channels(Conn3, 5), + ?assertEqual(3, count_connections_of_user(Config, Username)), + ?assertEqual(15, count_channels_of_user(Config, Username)), + + close_channels(Chans1 ++ Chans2 ++ Chans3), + ?awaitMatch(0, count_channels_of_user(Config, Username), 60000), + + close_connections([Conn1, Conn2, Conn3]), + ?awaitMatch(0, count_connections_of_user(Config, Username), 60000). + +cluster_single_user_connection_and_channel_count(Config) -> + Username = proplists:get_value(rmq_username, Config), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 0 andalso + count_channels_of_user(Config, Username) =:= 0 + end), + + [Conn1] = open_connections(Config, [0]), + _Chans1 = [_|_] = open_channels(Conn1, 5), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 1 andalso + count_channels_of_user(Config, Username) =:= 5 + end), + + close_connections([Conn1]), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 0 andalso + count_channels_of_user(Config, Username) =:= 0 + end), + + [Conn2] = open_connections(Config, [1]), + Chans2 = [_|_] = open_channels(Conn2, 5), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 1 andalso + count_channels_of_user(Config, Username) =:= 5 + end), + + [Conn3] = open_connections(Config, [0]), + Chans3 = [_|_] = open_channels(Conn3, 5), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 2 andalso + count_channels_of_user(Config, Username) =:= 10 + end), + + [Conn4] = open_connections(Config, [1]), + Chans4 = [_|_] = open_channels(Conn4, 5), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 3 andalso + count_channels_of_user(Config, Username) =:= 15 + end), + + close_channels(Chans2 ++ Chans3 ++ Chans4), + rabbit_ct_helpers:await_condition( + fun () -> + count_channels_of_user(Config, Username) =:= 0 + end), + + close_connections([Conn2, Conn3, Conn4]), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 0 + end). + +cluster_multiple_users_connection_and_channel_count(Config) -> + Username1 = <<"guest1">>, + Username2 = <<"guest2">>, + + set_up_user(Config, Username1), + set_up_user(Config, Username2), + + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username1) =:= 0 andalso + count_connections_of_user(Config, Username2) =:= 0 + end), + rabbit_ct_helpers:await_condition( + fun () -> + count_channels_of_user(Config, Username1) =:= 0 andalso + count_channels_of_user(Config, Username2) =:= 0 + end), + + [Conn1] = open_connections(Config, [{0, Username1}]), + _Chans1 = [_|_] = open_channels(Conn1, 5), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username1) =:= 1 andalso + count_channels_of_user(Config, Username1) =:= 5 + end), + close_connections([Conn1]), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username1) =:= 0 andalso + count_channels_of_user(Config, Username1) =:= 0 + end), + + [Conn2] = open_connections(Config, [{1, Username2}]), + Chans2 = [_|_] = open_channels(Conn2, 5), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username2) =:= 1 andalso + count_channels_of_user(Config, Username2) =:= 5 + end), + + [Conn3] = open_connections(Config, [{1, Username1}]), + Chans3 = [_|_] = open_channels(Conn3, 5), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username1) =:= 1 andalso + count_channels_of_user(Config, Username1) =:= 5 + end), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username2) =:= 1 andalso + count_channels_of_user(Config, Username2) =:= 5 + end), + + [Conn4] = open_connections(Config, [{0, Username1}]), + _Chans4 = [_|_] = open_channels(Conn4, 5), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username1) =:= 2 andalso + count_channels_of_user(Config, Username1) =:= 10 + end), + + close_connections([Conn4]), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username1) =:= 1 andalso + count_channels_of_user(Config, Username1) =:= 5 + end), + + [Conn5] = open_connections(Config, [{1, Username2}]), + Chans5 = [_|_] = open_channels(Conn5, 5), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username2) =:= 2 andalso + count_channels_of_user(Config, Username2) =:= 10 + end), + + [Conn6] = open_connections(Config, [{0, Username2}]), + Chans6 = [_|_] = open_channels(Conn6, 5), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username2) =:= 3 andalso + count_channels_of_user(Config, Username2) =:= 15 + end), + + close_channels(Chans2 ++ Chans3 ++ Chans5 ++ Chans6), + rabbit_ct_helpers:await_condition( + fun () -> + count_channels_of_user(Config, Username1) =:= 0 andalso + count_channels_of_user(Config, Username2) =:= 0 + end), + + close_connections([Conn2, Conn3, Conn5, Conn6]), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username1) =:= 0 andalso + count_connections_of_user(Config, Username2) =:= 0 + end), + + rabbit_ct_broker_helpers:delete_user(Config, Username1), + rabbit_ct_broker_helpers:delete_user(Config, Username2). + +cluster_node_restart_connection_and_channel_count(Config) -> + Username = proplists:get_value(rmq_username, Config), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 0 andalso + count_channels_of_user(Config, Username) =:= 0 + end), + + [Conn1] = open_connections(Config, [0]), + _Chans1 = [_|_] = open_channels(Conn1, 5), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 1 andalso + count_channels_of_user(Config, Username) =:= 5 + end), + close_connections([Conn1]), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 0 andalso + count_channels_of_user(Config, Username) =:= 0 + end), + + [Conn2] = open_connections(Config, [1]), + Chans2 = [_|_] = open_channels(Conn2, 5), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 1 andalso + count_channels_of_user(Config, Username) =:= 5 + end), + + [Conn3] = open_connections(Config, [0]), + Chans3 = [_|_] = open_channels(Conn3, 5), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 2 andalso + count_channels_of_user(Config, Username) =:= 10 + end), + + [Conn4] = open_connections(Config, [1]), + _Chans4 = [_|_] = open_channels(Conn4, 5), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 3 andalso + count_channels_of_user(Config, Username) =:= 15 + end), + + [Conn5] = open_connections(Config, [1]), + Chans5 = [_|_] = open_channels(Conn5, 5), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 4 andalso + count_channels_of_user(Config, Username) =:= 20 + end), + + rabbit_ct_broker_helpers:restart_broker(Config, 1), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 1 andalso + count_channels_of_user(Config, Username) =:= 5 + end), + + close_channels(Chans2 ++ Chans3 ++ Chans5), + rabbit_ct_helpers:await_condition( + fun () -> + count_channels_of_user(Config, Username) =:= 0 + end), + + close_connections([Conn2, Conn3, Conn4, Conn5]), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 0 + end). + +cluster_node_list_on_node(Config) -> + [A, B] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + rabbit_ct_helpers:await_condition( + fun () -> + length(all_connections(Config)) =:= 0 andalso + length(all_channels(Config)) =:= 0 andalso + length(connections_on_node(Config, 0)) =:= 0 andalso + length(channels_on_node(Config, 0)) =:= 0 + end), + + [Conn1] = open_connections(Config, [0]), + _Chans1 = [_|_] = open_channels(Conn1, 5), + [#tracked_connection{node = A}] = connections_on_node(Config, 0), + rabbit_ct_helpers:await_condition( + fun () -> + length([Ch || Ch <- channels_on_node(Config, 0), Ch#tracked_channel.node =:= A]) =:= 5 + end), + close_connections([Conn1]), + rabbit_ct_helpers:await_condition( + fun () -> + length(connections_on_node(Config, 0)) =:= 0 andalso + length(channels_on_node(Config, 0)) =:= 0 + end), + + [Conn2] = open_connections(Config, [1]), + _Chans2 = [_|_] = open_channels(Conn2, 5), + [#tracked_connection{node = B}] = connections_on_node(Config, 1), + rabbit_ct_helpers:await_condition( + fun () -> + length([Ch || Ch <- channels_on_node(Config, 1), Ch#tracked_channel.node =:= B]) =:= 5 + end), + + [Conn3] = open_connections(Config, [0]), + Chans3 = [_|_] = open_channels(Conn3, 5), + rabbit_ct_helpers:await_condition( + fun () -> + length(connections_on_node(Config, 0)) =:= 1 andalso + length(channels_on_node(Config, 0)) =:= 5 + end), + + [Conn4] = open_connections(Config, [1]), + _Chans4 = [_|_] = open_channels(Conn4, 5), + rabbit_ct_helpers:await_condition( + fun () -> + length(connections_on_node(Config, 1)) =:= 2 andalso + length(channels_on_node(Config, 1)) =:= 10 + end), + + close_connections([Conn4]), + rabbit_ct_helpers:await_condition( + fun () -> + length(connections_on_node(Config, 1)) =:= 1 andalso + length(channels_on_node(Config, 1)) =:= 5 + end), + + [Conn5] = open_connections(Config, [0]), + Chans5 = [_|_] = open_channels(Conn5, 5), + rabbit_ct_helpers:await_condition( + fun () -> + length(connections_on_node(Config, 0)) =:= 2 andalso + length(channels_on_node(Config, 0)) =:= 10 + end), + + rabbit_ct_broker_helpers:stop_broker(Config, 1), + await_running_node_refresh(Config, 0), + + rabbit_ct_helpers:await_condition( + fun () -> + length(all_connections(Config)) =:= 2 andalso + length(all_channels(Config)) =:= 10 + end), + + close_channels(Chans3 ++ Chans5), + rabbit_ct_helpers:await_condition( + fun () -> + length(all_channels(Config)) =:= 0 + end), + + close_connections([Conn3, Conn5]), + rabbit_ct_helpers:await_condition( + fun () -> + length(all_connections(Config)) =:= 0 + end), + + rabbit_ct_broker_helpers:start_broker(Config, 1). + +single_node_single_user_limit(Config) -> + single_node_single_user_limit_with(Config, 5, 25), + single_node_single_user_limit_with(Config, -1, -1). + +single_node_single_user_limit_with(Config, ConnLimit, ChLimit) -> + Username = proplists:get_value(rmq_username, Config), + set_user_connection_and_channel_limit(Config, Username, 3, 15), + + ?assertEqual(0, count_connections_of_user(Config, Username)), + ?assertEqual(0, count_channels_of_user(Config, Username)), + + [Conn1, Conn2, Conn3] = Conns1 = open_connections(Config, [0, 0, 0]), + [_Chans1, Chans2, Chans3] = [open_channels(Conn, 5) || Conn <- Conns1], + + %% we've crossed the limit + expect_that_client_connection_is_rejected(Config, 0), + expect_that_client_connection_is_rejected(Config, 0), + expect_that_client_connection_is_rejected(Config, 0), + expect_that_client_channel_is_rejected(Conn1), + + rabbit_ct_helpers:await_condition( + fun () -> + is_process_alive(Conn1) =:= false andalso + is_process_alive(Conn2) andalso + is_process_alive(Conn3) + end), + + set_user_connection_and_channel_limit(Config, Username, ConnLimit, ChLimit), + [Conn4, Conn5] = Conns2 = open_connections(Config, [0, 0]), + [Chans4, Chans5] = [open_channels(Conn, 5) || Conn <- Conns2], + + close_channels(Chans2 ++ Chans3 ++ Chans4 ++ Chans5), + rabbit_ct_helpers:await_condition( + fun () -> + count_channels_of_user(Config, Username) =:= 0 + end), + + close_connections([Conn1, Conn2, Conn3, Conn4, Conn5]), + ?awaitMatch(0, count_connections_of_user(Config, Username), 60000), + + set_user_connection_and_channel_limit(Config, Username, -1, -1). + +single_node_single_user_zero_limit(Config) -> + Username = proplists:get_value(rmq_username, Config), + set_user_connection_and_channel_limit(Config, Username, 0, 0), + + ?assertEqual(0, count_connections_of_user(Config, Username)), + ?assertEqual(0, count_channels_of_user(Config, Username)), + + %% with limit = 0 no connections are allowed + expect_that_client_connection_is_rejected(Config), + expect_that_client_connection_is_rejected(Config), + expect_that_client_connection_is_rejected(Config), + + %% with limit = 0 no channels are allowed + set_user_connection_and_channel_limit(Config, Username, 1, 0), + [ConnA] = open_connections(Config, [0]), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 1 + end), + expect_that_client_channel_is_rejected(ConnA), + rabbit_ct_helpers:await_condition( + fun () -> + is_process_alive(ConnA) =:= false andalso + count_connections_of_user(Config, Username) =:= 0 andalso + count_channels_of_user(Config, Username) =:= 0 + end), + + set_user_connection_and_channel_limit(Config, Username, -1, -1), + [Conn1, Conn2] = Conns1 = open_connections(Config, [0, 0]), + [Chans1, Chans2] = [open_channels(Conn, 5) || Conn <- Conns1], + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 2 andalso + count_channels_of_user(Config, Username) =:= 10 + end), + + close_channels(Chans1 ++ Chans2), + rabbit_ct_helpers:await_condition( + fun () -> + count_channels_of_user(Config, Username) =:= 0 + end), + + close_connections([Conn1, Conn2]), + ?awaitMatch(0, count_connections_of_user(Config, Username), 60000). + +single_node_single_user_clear_limits(Config) -> + Username = proplists:get_value(rmq_username, Config), + set_user_connection_and_channel_limit(Config, Username, 3, 15), + + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 0 andalso + count_channels_of_user(Config, Username) =:= 0 + end), + + [Conn1, Conn2, Conn3] = Conns1 = open_connections(Config, [0, 0, 0]), + [_Chans1, Chans2, Chans3] = [open_channels(Conn, 5) || Conn <- Conns1], + + %% we've crossed the limit + expect_that_client_connection_is_rejected(Config, 0), + expect_that_client_connection_is_rejected(Config, 0), + expect_that_client_connection_is_rejected(Config, 0), + expect_that_client_channel_is_rejected(Conn1), + + rabbit_ct_helpers:await_condition( + fun () -> + is_process_alive(Conn1) =:= false andalso + is_process_alive(Conn2) andalso + is_process_alive(Conn3) + end), + + %% reach limit again + [Conn4] = open_connections(Config, [{0, Username}]), + Chans4 = [_|_] = open_channels(Conn4, 5), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 3 andalso + count_channels_of_user(Config, Username) =:= 15 + end), + + clear_all_user_limits(Config, Username), + + [Conn5, Conn6, Conn7] = Conns2 = open_connections(Config, [0, 0, 0]), + [Chans5, Chans6, Chans7] = [open_channels(Conn, 5) || Conn <- Conns2], + + close_channels(Chans2 ++ Chans3 ++ Chans4 ++ Chans5 ++ Chans6 ++ Chans7), + rabbit_ct_helpers:await_condition( + fun () -> + count_channels_of_user(Config, Username) =:= 0 + end), + + close_connections([Conn2, Conn3, Conn4, Conn5, Conn6, Conn7]), + ?awaitMatch(0, count_connections_of_user(Config, Username), 5000), + + set_user_connection_and_channel_limit(Config, Username, -1, -1). + +single_node_multiple_users_clear_limits(Config) -> + Username1 = <<"guest1">>, + Username2 = <<"guest2">>, + + set_up_user(Config, Username1), + set_up_user(Config, Username2), + + set_user_connection_and_channel_limit(Config, Username1, 0, 0), + set_user_connection_and_channel_limit(Config, Username2, 0, 0), + + ?assertEqual(0, count_connections_of_user(Config, Username1)), + ?assertEqual(0, count_connections_of_user(Config, Username2)), + ?assertEqual(0, count_channels_of_user(Config, Username1)), + ?assertEqual(0, count_channels_of_user(Config, Username2)), + + %% with limit = 0 no connections are allowed + expect_that_client_connection_is_rejected(Config, 0, Username1), + expect_that_client_connection_is_rejected(Config, 0, Username2), + expect_that_client_connection_is_rejected(Config, 0, Username1), + + %% with limit = 0 no channels are allowed + set_user_connection_and_channel_limit(Config, Username1, 1, 0), + set_user_connection_and_channel_limit(Config, Username2, 1, 0), + [ConnA, ConnB] = open_connections(Config, [{0, Username1}, {0, Username2}]), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username1) =:= 1 + end), + expect_that_client_channel_is_rejected(ConnA), + expect_that_client_channel_is_rejected(ConnB), + + rabbit_ct_helpers:await_condition( + fun () -> + is_process_alive(ConnA) =:= false andalso + is_process_alive(ConnB) =:= false + end), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username1) =:= 0 andalso + count_connections_of_user(Config, Username2) =:= 0 + end), + rabbit_ct_helpers:await_condition( + fun () -> + count_channels_of_user(Config, Username1) =:= 0 andalso + count_channels_of_user(Config, Username2) =:= 0 + end), + + clear_all_user_limits(Config, Username1), + set_user_channel_limit_only(Config, Username2, -1), + set_user_connection_limit_only(Config, Username2, -1), + + [Conn1, Conn2] = Conns1 = open_connections(Config, [{0, Username1}, {0, Username1}]), + [Chans1, Chans2] = [open_channels(Conn, 5) || Conn <- Conns1], + + close_channels(Chans1 ++ Chans2), + rabbit_ct_helpers:await_condition( + fun () -> + count_channels_of_user(Config, Username1) =:= 0 andalso + count_channels_of_user(Config, Username2) =:= 0 + end), + + close_connections([Conn1, Conn2]), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username1) =:= 0 andalso + count_connections_of_user(Config, Username2) =:= 0 + end), + + set_user_connection_and_channel_limit(Config, Username1, -1, -1), + set_user_connection_and_channel_limit(Config, Username2, -1, -1). + +single_node_multiple_users_limit(Config) -> + Username1 = <<"guest1">>, + Username2 = <<"guest2">>, + + set_up_user(Config, Username1), + set_up_user(Config, Username2), + + set_user_connection_and_channel_limit(Config, Username1, 2, 10), + set_user_connection_and_channel_limit(Config, Username2, 2, 10), + + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username1) =:= 0 andalso + count_connections_of_user(Config, Username2) =:= 0 + end), + rabbit_ct_helpers:await_condition( + fun () -> + count_channels_of_user(Config, Username1) =:= 0 andalso + count_channels_of_user(Config, Username2) =:= 0 + end), + + [Conn1, Conn2, Conn3, Conn4] = Conns1 = open_connections(Config, [ + {0, Username1}, + {0, Username1}, + {0, Username2}, + {0, Username2}]), + + [_Chans1, Chans2, Chans3, Chans4] = [open_channels(Conn, 5) || Conn <- Conns1], + + %% we've crossed the limit + expect_that_client_connection_is_rejected(Config, 0, Username1), + expect_that_client_connection_is_rejected(Config, 0, Username2), + expect_that_client_channel_is_rejected(Conn1), + rabbit_ct_helpers:await_condition( + fun () -> + is_process_alive(Conn1) =:= false andalso + is_process_alive(Conn3) =:= true + end), + + [Conn5] = open_connections(Config, [0]), + Chans5 = [_|_] = open_channels(Conn5, 5), + + set_user_connection_and_channel_limit(Config, Username1, 5, 25), + set_user_connection_and_channel_limit(Config, Username2, -10, -50), + + [Conn6, Conn7, Conn8, Conn9, Conn10] = Conns2 = open_connections(Config, [ + {0, Username1}, + {0, Username1}, + {0, Username1}, + {0, Username2}, + {0, Username2}]), + + [Chans6, Chans7, Chans8, Chans9, Chans10] = [open_channels(Conn, 5) || Conn <- Conns2], + + close_channels(Chans2 ++ Chans3 ++ Chans4 ++ Chans5 ++ Chans6 ++ + Chans7 ++ Chans8 ++ Chans9 ++ Chans10), + rabbit_ct_helpers:await_condition( + fun () -> + count_channels_of_user(Config, Username1) =:= 0 andalso + count_channels_of_user(Config, Username2) =:= 0 + end), + + close_connections([Conn2, Conn3, Conn4, Conn5, Conn6, + Conn7, Conn8, Conn9, Conn10]), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username1) =:= 0 andalso + count_connections_of_user(Config, Username2) =:= 0 + end), + + set_user_connection_and_channel_limit(Config, Username1, -1, -1), + set_user_connection_and_channel_limit(Config, Username2, -1, -1), + + rabbit_ct_broker_helpers:delete_user(Config, Username1), + rabbit_ct_broker_helpers:delete_user(Config, Username2). + + +single_node_multiple_users_zero_limit(Config) -> + Username1 = <<"guest1">>, + Username2 = <<"guest2">>, + + set_up_user(Config, Username1), + set_up_user(Config, Username2), + + set_user_connection_and_channel_limit(Config, Username1, 0, 0), + set_user_connection_and_channel_limit(Config, Username2, 0, 0), + + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username1) =:= 0 andalso + count_connections_of_user(Config, Username2) =:= 0 + end), + rabbit_ct_helpers:await_condition( + fun () -> + count_channels_of_user(Config, Username1) =:= 0 andalso + count_channels_of_user(Config, Username2) =:= 0 + end), + + %% with limit = 0 no connections are allowed + expect_that_client_connection_is_rejected(Config, 0, Username1), + expect_that_client_connection_is_rejected(Config, 0, Username2), + expect_that_client_connection_is_rejected(Config, 0, Username1), + + %% with limit = 0 no channels are allowed + set_user_connection_and_channel_limit(Config, Username1, 1, 0), + set_user_connection_and_channel_limit(Config, Username2, 1, 0), + [ConnA, ConnB] = open_connections(Config, [{0, Username1}, {0, Username2}]), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username1) =:= 1 + end), + expect_that_client_channel_is_rejected(ConnA), + expect_that_client_channel_is_rejected(ConnB), + + rabbit_ct_helpers:await_condition( + fun () -> + is_process_alive(ConnA) =:= false andalso + is_process_alive(ConnB) =:= false + end), + + ?assertEqual(false, is_process_alive(ConnA)), + ?assertEqual(false, is_process_alive(ConnB)), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username1) =:= 0 andalso + count_connections_of_user(Config, Username2) =:= 0 + end), + rabbit_ct_helpers:await_condition( + fun () -> + count_channels_of_user(Config, Username1) =:= 0 andalso + count_channels_of_user(Config, Username2) =:= 0 + end), + + set_user_connection_and_channel_limit(Config, Username1, -1, -1), + [Conn1, Conn2] = Conns1 = open_connections(Config, [{0, Username1}, {0, Username1}]), + [Chans1, Chans2] = [open_channels(Conn, 5) || Conn <- Conns1], + + close_channels(Chans1 ++ Chans2), + rabbit_ct_helpers:await_condition( + fun () -> + count_channels_of_user(Config, Username1) =:= 0 andalso + count_channels_of_user(Config, Username2) =:= 0 + end), + + close_connections([Conn1, Conn2]), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username1) =:= 0 andalso + count_connections_of_user(Config, Username2) =:= 0 + end), + + set_user_connection_and_channel_limit(Config, Username1, -1, -1), + set_user_connection_and_channel_limit(Config, Username2, -1, -1). + + +cluster_single_user_limit(Config) -> + Username = proplists:get_value(rmq_username, Config), + set_user_connection_limit_only(Config, Username, 2), + set_user_channel_limit_only(Config, Username, 10), + + rabbit_ct_helpers:await_condition( + fun () -> + count_channels_of_user(Config, Username) =:= 0 andalso + count_channels_of_user(Config, Username) =:= 0 + end), + + %% here connections and channels are opened to different nodes + [Conn1, Conn2] = Conns1 = open_connections(Config, [{0, Username}, {1, Username}]), + [_Chans1, Chans2] = [open_channels(Conn, 5) || Conn <- Conns1], + + %% we've crossed the limit + expect_that_client_connection_is_rejected(Config, 0, Username), + expect_that_client_connection_is_rejected(Config, 1, Username), + expect_that_client_channel_is_rejected(Conn1), + rabbit_ct_helpers:await_condition( + fun () -> + is_process_alive(Conn1) =:= false andalso + is_process_alive(Conn2) =:= true + end), + + set_user_connection_and_channel_limit(Config, Username, 5, 25), + + [Conn3, Conn4] = Conns2 = open_connections(Config, [{0, Username}, {0, Username}]), + [Chans3, Chans4] = [open_channels(Conn, 5) || Conn <- Conns2], + + close_channels(Chans2 ++ Chans3 ++ Chans4), + ?awaitMatch(0, count_channels_of_user(Config, Username), 60000), + + close_connections([Conn2, Conn3, Conn4]), + ?awaitMatch(0, count_connections_of_user(Config, Username), 60000), + + set_user_connection_and_channel_limit(Config, Username, -1, -1). + +cluster_single_user_limit2(Config) -> + Username = proplists:get_value(rmq_username, Config), + set_user_connection_and_channel_limit(Config, Username, 2, 10), + + ?assertEqual(0, count_connections_of_user(Config, Username)), + ?assertEqual(0, count_channels_of_user(Config, Username)), + + %% here a limit is reached on one node first + [Conn1, Conn2] = Conns1 = open_connections(Config, [{0, Username}, {0, Username}]), + [_Chans1, Chans2] = [open_channels(Conn, 5) || Conn <- Conns1], + + %% we've crossed the limit + expect_that_client_connection_is_rejected(Config, 0, Username), + expect_that_client_connection_is_rejected(Config, 1, Username), + expect_that_client_channel_is_rejected(Conn1), + rabbit_ct_helpers:await_condition( + fun () -> + is_process_alive(Conn1) =:= false andalso + is_process_alive(Conn2) =:= true + end), + + set_user_connection_and_channel_limit(Config, Username, 5, 25), + + [Conn3, Conn4, Conn5, Conn6, {error, not_allowed}] = open_connections(Config, [ + {1, Username}, + {1, Username}, + {1, Username}, + {1, Username}, + {1, Username}]), + + [Chans3, Chans4, Chans5, Chans6, [{error, not_allowed}]] = + [open_channels(Conn, 1) || Conn <- [Conn3, Conn4, Conn5, Conn6, Conn1]], + + close_channels(Chans2 ++ Chans3 ++ Chans4 ++ Chans5 ++ Chans6), + rabbit_ct_helpers:await_condition( + fun () -> + count_channels_of_user(Config, Username) =:= 0 + end), + + close_connections([Conn2, Conn3, Conn4, Conn5, Conn6]), + ?awaitMatch(0, count_connections_of_user(Config, Username), 5000), + + set_user_connection_and_channel_limit(Config, Username, -1, -1). + + +cluster_single_user_zero_limit(Config) -> + Username = proplists:get_value(rmq_username, Config), + set_user_connection_and_channel_limit(Config, Username, 0, 0), + + ?assertEqual(0, count_connections_of_user(Config, Username)), + ?assertEqual(0, count_channels_of_user(Config, Username)), + + %% with limit = 0 no connections are allowed + expect_that_client_connection_is_rejected(Config, 0), + expect_that_client_connection_is_rejected(Config, 1), + expect_that_client_connection_is_rejected(Config, 0), + + %% with limit = 0 no channels are allowed + set_user_connection_and_channel_limit(Config, Username, 1, 0), + [ConnA] = open_connections(Config, [0]), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 1 + end), + expect_that_client_channel_is_rejected(ConnA), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 0 andalso + count_channels_of_user(Config, Username) =:= 0 + end), + ?assertEqual(false, is_process_alive(ConnA)), + + set_user_connection_and_channel_limit(Config, Username, -1, -1), + [Conn1, Conn2, Conn3, Conn4] = Conns1 = open_connections(Config, [0, 1, 0, 1]), + [Chans1, Chans2, Chans3, Chans4] = [open_channels(Conn, 5) || Conn <- Conns1], + + close_channels(Chans1 ++ Chans2 ++ Chans3 ++ Chans4), + rabbit_ct_helpers:await_condition( + fun () -> + count_channels_of_user(Config, Username) =:= 0 + end), + + close_connections([Conn1, Conn2, Conn3, Conn4]), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 0 + end), + + set_user_connection_and_channel_limit(Config, Username, -1, -1). + +cluster_single_user_clear_limits(Config) -> + Username = proplists:get_value(rmq_username, Config), + set_user_connection_and_channel_limit(Config, Username, 2, 10), + + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 0 andalso + count_channels_of_user(Config, Username) =:= 0 + end), + + %% here a limit is reached on one node first + [Conn1, Conn2] = Conns1 = open_connections(Config, [{0, Username}, {0, Username}]), + [_Chans1, Chans2] = [open_channels(Conn, 5) || Conn <- Conns1], + + %% we've crossed the limit + expect_that_client_connection_is_rejected(Config, 0, Username), + expect_that_client_connection_is_rejected(Config, 1, Username), + expect_that_client_channel_is_rejected(Conn1), + rabbit_ct_helpers:await_condition( + fun () -> + is_process_alive(Conn1) =:= false andalso + is_process_alive(Conn2) =:= true + end), + clear_all_user_limits(Config, Username), + + [Conn3, Conn4, Conn5, Conn6, Conn7] = open_connections(Config, [ + {1, Username}, + {1, Username}, + {1, Username}, + {1, Username}, + {1, Username}]), + + [Chans3, Chans4, Chans5, Chans6, Chans7] = + [open_channels(Conn, 1) || Conn <- [Conn3, Conn4, Conn5, Conn6, Conn7]], + + close_channels(Chans2 ++ Chans3 ++ Chans4 ++ Chans5 ++ Chans6 ++ Chans7), + rabbit_ct_helpers:await_condition( + fun () -> + count_channels_of_user(Config, Username) =:= 0 + end), + + close_connections([Conn2, Conn3, Conn4, Conn5, Conn6, Conn7]), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username) =:= 0 + end), + + set_user_connection_and_channel_limit(Config, Username, -1, -1). + +cluster_multiple_users_clear_limits(Config) -> + Username1 = <<"guest1">>, + Username2 = <<"guest2">>, + + set_up_user(Config, Username1), + set_up_user(Config, Username2), + + set_user_connection_and_channel_limit(Config, Username1, 0, 0), + set_user_connection_and_channel_limit(Config, Username2, 0, 0), + + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username1) =:= 0 andalso + count_connections_of_user(Config, Username2) =:= 0 + end), + rabbit_ct_helpers:await_condition( + fun () -> + count_channels_of_user(Config, Username1) =:= 0 andalso + count_channels_of_user(Config, Username2) =:= 0 + end), + + %% with limit = 0 no connections are allowed + expect_that_client_connection_is_rejected(Config, 0, Username1), + expect_that_client_connection_is_rejected(Config, 0, Username2), + expect_that_client_connection_is_rejected(Config, 1, Username1), + expect_that_client_connection_is_rejected(Config, 1, Username2), + + %% with limit = 0 no channels are allowed + set_user_connection_and_channel_limit(Config, Username1, 1, 0), + set_user_connection_and_channel_limit(Config, Username2, 1, 0), + [ConnA, ConnB] = open_connections(Config, [{0, Username1}, {1, Username2}]), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username1) =:= 1 andalso + count_connections_of_user(Config, Username2) =:= 1 + end), + expect_that_client_channel_is_rejected(ConnA), + + rabbit_ct_helpers:await_condition( + fun () -> + is_process_alive(ConnA) =:= false andalso + is_process_alive(ConnB) =:= true + end), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username1) =:= 0 andalso + count_connections_of_user(Config, Username2) =:= 1 + end), + rabbit_ct_helpers:await_condition( + fun () -> + count_channels_of_user(Config, Username1) =:= 0 andalso + count_channels_of_user(Config, Username2) =:= 0 + end), + close_connections([ConnB]), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username2) =:= 0 andalso + count_channels_of_user(Config, Username2) =:= 0 + end), + ?assertEqual(false, is_process_alive(ConnB)), + + clear_all_user_limits(Config, Username1), + clear_all_user_limits(Config, Username2), + + [Conn1, Conn2, Conn3, Conn4] = Conns1 = open_connections(Config, [ + {0, Username1}, + {0, Username2}, + {1, Username1}, + {1, Username2}]), + + [Chans1, Chans2, Chans3, Chans4] = [open_channels(Conn, 5) || Conn <- Conns1], + + close_channels(Chans1 ++ Chans2 ++ Chans3 ++ Chans4), + rabbit_ct_helpers:await_condition( + fun () -> + count_channels_of_user(Config, Username1) =:= 0 andalso + count_channels_of_user(Config, Username2) =:= 0 + end), + + close_connections([Conn1, Conn2, Conn3, Conn4]), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username1) =:= 0 andalso + count_connections_of_user(Config, Username2) =:= 0 + end), + + set_user_connection_and_channel_limit(Config, Username1, -1, -1), + set_user_connection_and_channel_limit(Config, Username2, -1, -1). + +cluster_multiple_users_zero_limit(Config) -> + Username1 = <<"guest1">>, + Username2 = <<"guest2">>, + + set_up_user(Config, Username1), + set_up_user(Config, Username2), + + set_user_connection_and_channel_limit(Config, Username1, 0, 0), + set_user_connection_and_channel_limit(Config, Username2, 0, 0), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username1) =:= 0 andalso + count_connections_of_user(Config, Username2) =:= 0 + end), + rabbit_ct_helpers:await_condition( + fun () -> + count_channels_of_user(Config, Username1) =:= 0 andalso + count_channels_of_user(Config, Username2) =:= 0 + end), + + %% with limit = 0 no connections are allowed + expect_that_client_connection_is_rejected(Config, 0, Username1), + expect_that_client_connection_is_rejected(Config, 0, Username2), + expect_that_client_connection_is_rejected(Config, 1, Username1), + expect_that_client_connection_is_rejected(Config, 1, Username2), + + %% with limit = 0 no channels are allowed + set_user_connection_and_channel_limit(Config, Username1, 1, 0), + set_user_connection_and_channel_limit(Config, Username2, 1, 0), + [ConnA, ConnB] = open_connections(Config, [{0, Username1}, {1, Username2}]), + + expect_that_client_channel_is_rejected(ConnA), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username1) =:= 0 andalso + count_connections_of_user(Config, Username2) =:= 1 + end), + rabbit_ct_helpers:await_condition( + fun () -> + count_channels_of_user(Config, Username1) =:= 0 andalso + count_channels_of_user(Config, Username2) =:= 0 + end), + ?assertEqual(false, is_process_alive(ConnA)), + ?assertEqual(true, is_process_alive(ConnB)), + close_connections([ConnB]), + rabbit_ct_helpers:await_condition( + fun () -> + count_connections_of_user(Config, Username2) =:= 0 andalso + count_channels_of_user(Config, Username2) =:= 0 + end), + ?assertEqual(false, is_process_alive(ConnB)), + + set_user_connection_and_channel_limit(Config, Username1, -1, -1), + set_user_connection_and_channel_limit(Config, Username2, -1, -1), + + [Conn1, Conn2, Conn3, Conn4] = Conns1 = open_connections(Config, [ + {0, Username1}, + {0, Username2}, + {1, Username1}, + {1, Username2}]), + + [Chans1, Chans2, Chans3, Chans4] = [open_channels(Conn, 5) || Conn <- Conns1], + + close_channels(Chans1 ++ Chans2 ++ Chans3 ++ Chans4), + ?awaitMatch(0, count_channels_of_user(Config, Username1), 60000), + ?awaitMatch(0, count_channels_of_user(Config, Username2), 60000), + + close_connections([Conn1, Conn2, Conn3, Conn4]), + ?awaitMatch(0, count_connections_of_user(Config, Username1), 60000), + ?awaitMatch(0, count_connections_of_user(Config, Username2), 60000), + + set_user_connection_and_channel_limit(Config, Username1, -1, -1), + set_user_connection_and_channel_limit(Config, Username2, -1, -1). + +%% ------------------------------------------------------------------- +%% Helpers +%% ------------------------------------------------------------------- + +open_connections(Config, NodesAndUsers) -> + % Randomly select connection type + OpenConnectionFun = case ?config(connection_type, Config) of + network -> open_unmanaged_connection; + direct -> open_unmanaged_connection_direct + end, + Conns = lists:map(fun + ({Node, User}) -> + rabbit_ct_client_helpers:OpenConnectionFun(Config, Node, + User, User); + (Node) -> + rabbit_ct_client_helpers:OpenConnectionFun(Config, Node) + end, NodesAndUsers), + timer:sleep(100), + Conns. + +close_connections(Conns) -> + lists:foreach(fun + (Conn) -> + rabbit_ct_client_helpers:close_connection(Conn) + end, Conns). + +open_channels(Conn, N) -> + [open_channel(Conn) || _ <- lists:seq(1, N)]. + +open_channel(Conn) when is_pid(Conn) -> + try amqp_connection:open_channel(Conn) of + {ok, Ch} -> Ch + catch + _:_Error -> {error, not_allowed} + end. + +close_channels(Channels = [_|_]) -> + [rabbit_ct_client_helpers:close_channel(Ch) || Ch <- Channels]. + +count_connections_of_user(Config, Username) -> + count_connections_in(Config, Username, 0). +count_connections_in(Config, Username, NodeIndex) -> + count_user_tracked_items(Config, NodeIndex, rabbit_connection_tracking, Username). + +count_channels_of_user(Config, Username) -> + count_channels_in(Config, Username, 0). +count_channels_in(Config, Username, NodeIndex) -> + count_user_tracked_items(Config, NodeIndex, rabbit_channel_tracking, Username). + +count_user_tracked_items(Config, NodeIndex, TrackingMod, Username) -> + rabbit_ct_broker_helpers:rpc(Config, NodeIndex, + TrackingMod, + count_tracked_items_in, [{user, Username}]). + +connections_in(Config, Username) -> + connections_in(Config, 0, Username). +connections_in(Config, NodeIndex, Username) -> + tracked_list_of_user(Config, NodeIndex, rabbit_connection_tracking, Username). + +channels_in(Config, Username) -> + channels_in(Config, 0, Username). +channels_in(Config, NodeIndex, Username) -> + tracked_list_of_user(Config, NodeIndex, rabbit_channel_tracking, Username). + +tracked_list_of_user(Config, NodeIndex, TrackingMod, Username) -> + rabbit_ct_broker_helpers:rpc(Config, NodeIndex, + TrackingMod, + list_of_user, [Username]). + +connections_on_node(Config) -> + connections_on_node(Config, 0). +connections_on_node(Config, NodeIndex) -> + Node = rabbit_ct_broker_helpers:get_node_config(Config, NodeIndex, nodename), + tracked_items_on_node(Config, NodeIndex, rabbit_connection_tracking, Node). + +channels_on_node(Config) -> + channels_on_node(Config, 0). +channels_on_node(Config, NodeIndex) -> + Node = rabbit_ct_broker_helpers:get_node_config(Config, NodeIndex, nodename), + tracked_items_on_node(Config, NodeIndex, rabbit_channel_tracking, Node). + +tracked_items_on_node(Config, NodeIndex, TrackingMod, NodeForListing) -> + rabbit_ct_broker_helpers:rpc(Config, NodeIndex, + TrackingMod, + list_on_node, [NodeForListing]). + +all_connections(Config) -> + all_connections(Config, 0). +all_connections(Config, NodeIndex) -> + all_tracked_items(Config, NodeIndex, rabbit_connection_tracking). + +all_channels(Config) -> + all_channels(Config, 0). +all_channels(Config, NodeIndex) -> + all_tracked_items(Config, NodeIndex, rabbit_channel_tracking). + +all_tracked_items(Config, NodeIndex, TrackingMod) -> + rabbit_ct_broker_helpers:rpc(Config, NodeIndex, + TrackingMod, + list, []). + +set_up_user(Config, Username) -> + VHost = proplists:get_value(rmq_vhost, Config), + rabbit_ct_broker_helpers:add_user(Config, Username), + rabbit_ct_broker_helpers:set_full_permissions(Config, Username, VHost), + set_user_connection_and_channel_limit(Config, Username, -1, -1). + +set_user_connection_and_channel_limit(Config, Username, ConnLimit, ChLimit) -> + set_user_connection_and_channel_limit(Config, 0, Username, ConnLimit, ChLimit). + +set_user_connection_and_channel_limit(Config, NodeIndex, Username, ConnLimit, ChLimit) -> + Node = rabbit_ct_broker_helpers:get_node_config( + Config, NodeIndex, nodename), + ok = rabbit_ct_broker_helpers:control_action( + set_user_limits, Node, [rabbit_data_coercion:to_list(Username)] ++ + ["{\"max-connections\": " ++ integer_to_list(ConnLimit) ++ "," ++ + " \"max-channels\": " ++ integer_to_list(ChLimit) ++ "}"]). + +set_user_connection_limit_only(Config, Username, ConnLimit) -> + set_user_connection_limit_only(Config, 0, Username, ConnLimit). + +set_user_connection_limit_only(Config, NodeIndex, Username, ConnLimit) -> + Node = rabbit_ct_broker_helpers:get_node_config( + Config, NodeIndex, nodename), + ok = rabbit_ct_broker_helpers:control_action( + set_user_limits, Node, [rabbit_data_coercion:to_list(Username)] ++ + ["{\"max-connections\": " ++ integer_to_list(ConnLimit) ++ "}"]). + +set_user_channel_limit_only(Config, Username, ChLimit) -> + set_user_channel_limit_only(Config, 0, Username, ChLimit). + +set_user_channel_limit_only(Config, NodeIndex, Username, ChLimit) -> + Node = rabbit_ct_broker_helpers:get_node_config( + Config, NodeIndex, nodename), + ok = rabbit_ct_broker_helpers:control_action( + set_user_limits, Node, [rabbit_data_coercion:to_list(Username)] ++ + ["{\"max-channels\": " ++ integer_to_list(ChLimit) ++ "}"]). + +clear_all_user_limits(Config, Username) -> + clear_all_user_limits(Config, 0, Username). +clear_all_user_limits(Config, NodeIndex, Username) -> + Node = rabbit_ct_broker_helpers:get_node_config( + Config, NodeIndex, nodename), + ok = rabbit_ct_broker_helpers:control_action( + clear_user_limits, Node, [rabbit_data_coercion:to_list(Username), "all"]). + +await_running_node_refresh(_Config, _NodeIndex) -> + timer:sleep(250). + +expect_that_client_connection_is_rejected(Config) -> + expect_that_client_connection_is_rejected(Config, 0). + +expect_that_client_connection_is_rejected(Config, NodeIndex) -> + {error, not_allowed} = + rabbit_ct_client_helpers:open_unmanaged_connection(Config, NodeIndex). + +expect_that_client_connection_is_rejected(Config, NodeIndex, User) -> + {error, not_allowed} = + rabbit_ct_client_helpers:open_unmanaged_connection(Config, NodeIndex, User, User). + +expect_that_client_channel_is_rejected(Conn) -> + {error, not_allowed} = open_channel(Conn). diff --git a/deps/rabbit/test/per_user_connection_channel_limit_partitions_SUITE.erl b/deps/rabbit/test/per_user_connection_channel_limit_partitions_SUITE.erl new file mode 100644 index 0000000000..8af68f0112 --- /dev/null +++ b/deps/rabbit/test/per_user_connection_channel_limit_partitions_SUITE.erl @@ -0,0 +1,182 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(per_user_connection_channel_limit_partitions_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-compile(export_all). + +-import(rabbit_ct_client_helpers, [open_unmanaged_connection/2 + ]). + +all() -> + [ + {group, net_ticktime_1} + ]. + +groups() -> + [ + {net_ticktime_1, [], [ + cluster_full_partition_with_autoheal + ]} + ]. + +suite() -> + [ + %% If a test hangs, no need to wait for 30 minutes. + {timetrap, {minutes, 8}} + ]. + +%% see partitions_SUITE +-define(DELAY, 12000). + +%% ------------------------------------------------------------------- +%% Testsuite setup/teardown. +%% ------------------------------------------------------------------- + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps( + Config, [fun rabbit_ct_broker_helpers:configure_dist_proxy/1]). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(net_ticktime_1 = Group, Config) -> + Config1 = rabbit_ct_helpers:set_config(Config, [{net_ticktime, 1}]), + init_per_multinode_group(Group, Config1, 3). + +init_per_multinode_group(Group, Config, NodeCount) -> + Suffix = rabbit_ct_helpers:testcase_absname(Config, "", "-"), + Config1 = rabbit_ct_helpers:set_config(Config, [ + {rmq_nodes_count, NodeCount}, + {rmq_nodename_suffix, Suffix} + ]), + Config2 = rabbit_ct_helpers:run_steps( + Config1, rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()), + EnableFF = rabbit_ct_broker_helpers:enable_feature_flag( + Config2, user_limits), + case EnableFF of + ok -> + Config2; + Skip -> + end_per_group(Group, Config2), + Skip + end. + +end_per_group(_Group, Config) -> + rabbit_ct_helpers:run_steps(Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()). + +init_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_started(Config, Testcase). + +end_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_finished(Config, Testcase). + +%% ------------------------------------------------------------------- +%% Test cases. +%% ------------------------------------------------------------------- + +cluster_full_partition_with_autoheal(Config) -> + Username = proplists:get_value(rmq_username, Config), + rabbit_ct_broker_helpers:set_partition_handling_mode_globally(Config, autoheal), + + ?assertEqual(0, count_connections_in(Config, Username)), + [A, B, C] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + %% 6 connections, 2 per node + Conn1 = open_unmanaged_connection(Config, A), + Conn2 = open_unmanaged_connection(Config, A), + Conn3 = open_unmanaged_connection(Config, B), + Conn4 = open_unmanaged_connection(Config, B), + Conn5 = open_unmanaged_connection(Config, C), + Conn6 = open_unmanaged_connection(Config, C), + + _Chans1 = [_|_] = open_channels(Conn1, 5), + _Chans3 = [_|_] = open_channels(Conn3, 5), + _Chans5 = [_|_] = open_channels(Conn5, 5), + wait_for_count_connections_in(Config, Username, 6, 60000), + ?assertEqual(15, count_channels_in(Config, Username)), + + %% B drops off the network, non-reachable by either A or C + rabbit_ct_broker_helpers:block_traffic_between(A, B), + rabbit_ct_broker_helpers:block_traffic_between(B, C), + timer:sleep(?DELAY), + + %% A and C are still connected, so 4 connections are tracked + %% All connections to B are dropped + wait_for_count_connections_in(Config, Username, 4, 60000), + ?assertEqual(10, count_channels_in(Config, Username)), + + rabbit_ct_broker_helpers:allow_traffic_between(A, B), + rabbit_ct_broker_helpers:allow_traffic_between(B, C), + timer:sleep(?DELAY), + + %% during autoheal B's connections were dropped + wait_for_count_connections_in(Config, Username, 4, 60000), + ?assertEqual(10, count_channels_in(Config, Username)), + + lists:foreach(fun (Conn) -> + (catch rabbit_ct_client_helpers:close_connection(Conn)) + end, [Conn1, Conn2, Conn3, Conn4, + Conn5, Conn6]), + ?assertEqual(0, count_connections_in(Config, Username)), + ?assertEqual(0, count_channels_in(Config, Username)), + + passed. + +%% ------------------------------------------------------------------- +%% Helpers +%% ------------------------------------------------------------------- + +wait_for_count_connections_in(Config, Username, Expected, Time) when Time =< 0 -> + ?assertMatch(Connections when length(Connections) == Expected, + connections_in(Config, Username)); +wait_for_count_connections_in(Config, Username, Expected, Time) -> + case connections_in(Config, Username) of + Connections when length(Connections) == Expected -> + ok; + _ -> + Sleep = 3000, + timer:sleep(Sleep), + wait_for_count_connections_in(Config, Username, Expected, Time - Sleep) + end. + +open_channels(Conn, N) -> + [begin + {ok, Ch} = amqp_connection:open_channel(Conn), + Ch + end || _ <- lists:seq(1, N)]. + +count_connections_in(Config, Username) -> + length(connections_in(Config, Username)). + +connections_in(Config, Username) -> + connections_in(Config, 0, Username). +connections_in(Config, NodeIndex, Username) -> + tracked_list_of_user(Config, NodeIndex, rabbit_connection_tracking, Username). + +count_channels_in(Config, Username) -> + Channels = channels_in(Config, Username), + length([Ch || Ch = #tracked_channel{username = Username0} <- Channels, + Username =:= Username0]). + +channels_in(Config, Username) -> + channels_in(Config, 0, Username). +channels_in(Config, NodeIndex, Username) -> + tracked_list_of_user(Config, NodeIndex, rabbit_channel_tracking, Username). + +tracked_list_of_user(Config, NodeIndex, TrackingMod, Username) -> + rabbit_ct_broker_helpers:rpc(Config, NodeIndex, + TrackingMod, + list_of_user, [Username]). diff --git a/deps/rabbit/test/per_user_connection_channel_tracking_SUITE.erl b/deps/rabbit/test/per_user_connection_channel_tracking_SUITE.erl new file mode 100644 index 0000000000..8b4bd91d09 --- /dev/null +++ b/deps/rabbit/test/per_user_connection_channel_tracking_SUITE.erl @@ -0,0 +1,850 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(per_user_connection_channel_tracking_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("rabbitmq_ct_helpers/include/rabbit_assert.hrl"). + +-compile(export_all). + +all() -> + [ + {group, cluster_size_1_network}, + {group, cluster_size_2_network}, + {group, cluster_size_1_direct}, + {group, cluster_size_2_direct} + ]. + +groups() -> + ClusterSize1Tests = [ + single_node_user_connection_channel_tracking, + single_node_user_deletion, + single_node_vhost_down_mimic, + single_node_vhost_deletion + ], + ClusterSize2Tests = [ + cluster_user_deletion, + cluster_vhost_down_mimic, + cluster_vhost_deletion, + cluster_node_removed + ], + [ + {cluster_size_1_network, [], ClusterSize1Tests}, + {cluster_size_2_network, [], ClusterSize2Tests}, + {cluster_size_1_direct, [], ClusterSize1Tests}, + {cluster_size_2_direct, [], ClusterSize2Tests} + ]. + +suite() -> + [ + %% If a test hangs, no need to wait for 30 minutes. + {timetrap, {minutes, 8}} + ]. + +%% ------------------------------------------------------------------- +%% Testsuite setup/teardown. +%% ------------------------------------------------------------------- + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(cluster_size_1_network, Config) -> + Config1 = rabbit_ct_helpers:set_config(Config, [{connection_type, network}]), + init_per_multinode_group(cluster_size_1_network, Config1, 1); +init_per_group(cluster_size_2_network, Config) -> + Config1 = rabbit_ct_helpers:set_config(Config, [{connection_type, network}]), + init_per_multinode_group(cluster_size_2_network, Config1, 2); +init_per_group(cluster_size_1_direct, Config) -> + Config1 = rabbit_ct_helpers:set_config(Config, [{connection_type, direct}]), + init_per_multinode_group(cluster_size_1_direct, Config1, 1); +init_per_group(cluster_size_2_direct, Config) -> + Config1 = rabbit_ct_helpers:set_config(Config, [{connection_type, direct}]), + init_per_multinode_group(cluster_size_2_direct, Config1, 2). + +init_per_multinode_group(Group, Config, NodeCount) -> + Suffix = rabbit_ct_helpers:testcase_absname(Config, "", "-"), + Config1 = rabbit_ct_helpers:set_config(Config, [ + {rmq_nodes_count, NodeCount}, + {rmq_nodename_suffix, Suffix} + ]), + Config2 = rabbit_ct_helpers:run_steps( + Config1, rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()), + EnableFF = rabbit_ct_broker_helpers:enable_feature_flag( + Config2, user_limits), + case EnableFF of + ok -> + Config2; + Skip -> + end_per_group(Group, Config2), + Skip + end. + +end_per_group(_Group, Config) -> + rabbit_ct_helpers:run_steps(Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()). + +init_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_started(Config, Testcase), + clear_all_connection_tracking_tables(Config), + clear_all_channel_tracking_tables(Config), + Config. + +end_per_testcase(Testcase, Config) -> + clear_all_connection_tracking_tables(Config), + clear_all_channel_tracking_tables(Config), + rabbit_ct_helpers:testcase_finished(Config, Testcase). + +clear_all_connection_tracking_tables(Config) -> + [rabbit_ct_broker_helpers:rpc(Config, + N, + rabbit_connection_tracking, + clear_tracking_tables, + []) || N <- rabbit_ct_broker_helpers:get_node_configs(Config, nodename)]. + +clear_all_channel_tracking_tables(Config) -> + [rabbit_ct_broker_helpers:rpc(Config, + N, + rabbit_channel_tracking, + clear_tracking_tables, + []) || N <- rabbit_ct_broker_helpers:get_node_configs(Config, nodename)]. + +%% ------------------------------------------------------------------- +%% Test cases. +%% ------------------------------------------------------------------- +single_node_user_connection_channel_tracking(Config) -> + Username = proplists:get_value(rmq_username, Config), + Username2 = <<"guest2">>, + + Vhost = proplists:get_value(rmq_vhost, Config), + + rabbit_ct_broker_helpers:add_user(Config, Username2), + rabbit_ct_broker_helpers:set_full_permissions(Config, Username2, Vhost), + + ?assertEqual(0, count_connections_in(Config, Username)), + ?assertEqual(0, count_connections_in(Config, Username2)), + ?assertEqual(0, count_channels_in(Config, Username)), + ?assertEqual(0, count_channels_in(Config, Username2)), + ?assertEqual(0, tracked_user_connection_count(Config, Username)), + ?assertEqual(0, tracked_user_connection_count(Config, Username2)), + ?assertEqual(0, tracked_user_channel_count(Config, Username)), + ?assertEqual(0, tracked_user_channel_count(Config, Username2)), + + [Conn1] = open_connections(Config, [0]), + [Chan1] = open_channels(Conn1, 1), + [#tracked_connection{username = Username}] = connections_in(Config, Username), + [#tracked_channel{username = Username}] = channels_in(Config, Username), + ?assertEqual(true, is_process_alive(Conn1)), + ?assertEqual(true, is_process_alive(Chan1)), + close_channels([Chan1]), + ?awaitMatch(0, count_channels_in(Config, Username), 20000), + ?awaitMatch(0, tracked_user_channel_count(Config, Username), 20000), + ?awaitMatch(false, is_process_alive(Chan1), 20000), + close_connections([Conn1]), + ?awaitMatch(0, length(connections_in(Config, Username)), 20000), + ?awaitMatch(0, tracked_user_connection_count(Config, Username), 20000), + ?awaitMatch(false, is_process_alive(Conn1), 20000), + + [Conn2] = open_connections(Config, [{0, Username2}]), + Chans2 = [_|_] = open_channels(Conn2, 5), + timer:sleep(100), + [#tracked_connection{username = Username2}] = connections_in(Config, Username2), + ?assertEqual(5, count_channels_in(Config, Username2)), + ?assertEqual(1, tracked_user_connection_count(Config, Username2)), + ?assertEqual(5, tracked_user_channel_count(Config, Username2)), + ?assertEqual(true, is_process_alive(Conn2)), + [?assertEqual(true, is_process_alive(Ch)) || Ch <- Chans2], + + [Conn3] = open_connections(Config, [0]), + Chans3 = [_|_] = open_channels(Conn3, 5), + [#tracked_connection{username = Username}] = connections_in(Config, Username), + ?assertEqual(5, count_channels_in(Config, Username)), + ?assertEqual(1, tracked_user_connection_count(Config, Username)), + ?assertEqual(5, tracked_user_channel_count(Config, Username)), + ?assertEqual(true, is_process_alive(Conn3)), + [?assertEqual(true, is_process_alive(Ch)) || Ch <- Chans3], + + [Conn4] = open_connections(Config, [0]), + Chans4 = [_|_] = open_channels(Conn4, 5), + ?assertEqual(2, tracked_user_connection_count(Config, Username)), + ?assertEqual(10, tracked_user_channel_count(Config, Username)), + ?assertEqual(true, is_process_alive(Conn4)), + [?assertEqual(true, is_process_alive(Ch)) || Ch <- Chans4], + kill_connections([Conn4]), + [#tracked_connection{username = Username}] = connections_in(Config, Username), + ?awaitMatch(5, count_channels_in(Config, Username), 20000), + ?awaitMatch(1, tracked_user_connection_count(Config, Username), 20000), + ?awaitMatch(5, tracked_user_channel_count(Config, Username), 20000), + ?assertEqual(false, is_process_alive(Conn4)), + [?assertEqual(false, is_process_alive(Ch)) || Ch <- Chans4], + + [Conn5] = open_connections(Config, [0]), + Chans5 = [_|_] = open_channels(Conn5, 7), + [Username, Username] = + lists:map(fun (#tracked_connection{username = U}) -> U end, + connections_in(Config, Username)), + ?assertEqual(12, count_channels_in(Config, Username)), + ?assertEqual(12, tracked_user_channel_count(Config, Username)), + ?assertEqual(2, tracked_user_connection_count(Config, Username)), + ?assertEqual(true, is_process_alive(Conn5)), + [?assertEqual(true, is_process_alive(Ch)) || Ch <- Chans5], + + close_channels(Chans2 ++ Chans3 ++ Chans5), + ?awaitMatch(0, length(all_channels(Config)), 20000), + ?awaitMatch(0, tracked_user_channel_count(Config, Username), 20000), + ?awaitMatch(0, tracked_user_channel_count(Config, Username2), 20000), + + close_connections([Conn2, Conn3, Conn5]), + rabbit_ct_broker_helpers:delete_user(Config, Username2), + ?awaitMatch(0, tracked_user_connection_count(Config, Username), 20000), + ?awaitMatch(0, tracked_user_connection_count(Config, Username2), 20000), + ?awaitMatch(0, length(all_connections(Config)), 20000). + +single_node_user_deletion(Config) -> + set_tracking_execution_timeout(Config, 100), + + Username = proplists:get_value(rmq_username, Config), + Username2 = <<"guest2">>, + + Vhost = proplists:get_value(rmq_vhost, Config), + + rabbit_ct_broker_helpers:add_user(Config, Username2), + rabbit_ct_broker_helpers:set_full_permissions(Config, Username2, Vhost), + + ?assertEqual(100, get_tracking_execution_timeout(Config)), + + ?assertEqual(0, count_connections_in(Config, Username)), + ?assertEqual(0, count_connections_in(Config, Username2)), + ?assertEqual(0, count_channels_in(Config, Username)), + ?assertEqual(0, count_channels_in(Config, Username2)), + ?assertEqual(0, tracked_user_connection_count(Config, Username)), + ?assertEqual(0, tracked_user_connection_count(Config, Username2)), + ?assertEqual(0, tracked_user_channel_count(Config, Username)), + ?assertEqual(0, tracked_user_channel_count(Config, Username2)), + + [Conn1] = open_connections(Config, [0]), + Chans1 = [_|_] = open_channels(Conn1, 5), + ?assertEqual(1, count_connections_in(Config, Username)), + ?assertEqual(5, count_channels_in(Config, Username)), + ?assertEqual(1, tracked_user_connection_count(Config, Username)), + ?assertEqual(5, tracked_user_channel_count(Config, Username)), + ?assertEqual(true, is_process_alive(Conn1)), + [?assertEqual(true, is_process_alive(Ch)) || Ch <- Chans1], + + [Conn2] = open_connections(Config, [{0, Username2}]), + Chans2 = [_|_] = open_channels(Conn2, 5), + ?assertEqual(1, count_connections_in(Config, Username2)), + ?assertEqual(5, count_channels_in(Config, Username2)), + ?assertEqual(1, tracked_user_connection_count(Config, Username2)), + ?assertEqual(5, tracked_user_channel_count(Config, Username2)), + ?assertEqual(true, is_process_alive(Conn2)), + [?assertEqual(true, is_process_alive(Ch)) || Ch <- Chans2], + + ?assertEqual(true, exists_in_tracked_connection_per_user_table(Config, Username2)), + ?assertEqual(true, exists_in_tracked_channel_per_user_table(Config, Username2)), + + rabbit_ct_broker_helpers:delete_user(Config, Username2), + timer:sleep(100), + ?assertEqual(0, count_connections_in(Config, Username2)), + ?assertEqual(0, count_channels_in(Config, Username2)), + ?assertEqual(0, tracked_user_connection_count(Config, Username2)), + ?assertEqual(0, tracked_user_channel_count(Config, Username2)), + ?assertEqual(false, is_process_alive(Conn2)), + [?assertEqual(false, is_process_alive(Ch)) || Ch <- Chans2], + + %% ensure vhost entry is cleared after 'tracking_execution_timeout' + ?awaitMatch(false, exists_in_tracked_connection_per_user_table(Config, Username2), 20000), + ?awaitMatch(false, exists_in_tracked_channel_per_user_table(Config, Username2), 20000), + + ?assertEqual(1, count_connections_in(Config, Username)), + ?assertEqual(5, count_channels_in(Config, Username)), + ?assertEqual(1, tracked_user_connection_count(Config, Username)), + ?assertEqual(5, tracked_user_channel_count(Config, Username)), + ?assertEqual(true, is_process_alive(Conn1)), + [?assertEqual(true, is_process_alive(Ch)) || Ch <- Chans1], + + close_channels(Chans1), + ?awaitMatch(0, count_channels_in(Config, Username), 20000), + ?awaitMatch(0, tracked_user_channel_count(Config, Username), 20000), + + close_connections([Conn1]), + ?awaitMatch(0, count_connections_in(Config, Username), 20000), + ?awaitMatch(0, tracked_user_connection_count(Config, Username), 20000). + +single_node_vhost_deletion(Config) -> + set_tracking_execution_timeout(Config, 100), + + Username = proplists:get_value(rmq_username, Config), + Username2 = <<"guest2">>, + + Vhost = proplists:get_value(rmq_vhost, Config), + + rabbit_ct_broker_helpers:add_user(Config, Username2), + rabbit_ct_broker_helpers:set_full_permissions(Config, Username2, Vhost), + + ?assertEqual(100, get_tracking_execution_timeout(Config)), + + ?assertEqual(0, count_connections_in(Config, Username)), + ?assertEqual(0, count_connections_in(Config, Username2)), + ?assertEqual(0, count_channels_in(Config, Username)), + ?assertEqual(0, count_channels_in(Config, Username2)), + ?assertEqual(0, tracked_user_connection_count(Config, Username)), + ?assertEqual(0, tracked_user_connection_count(Config, Username2)), + ?assertEqual(0, tracked_user_channel_count(Config, Username)), + ?assertEqual(0, tracked_user_channel_count(Config, Username2)), + + [Conn1] = open_connections(Config, [0]), + Chans1 = [_|_] = open_channels(Conn1, 5), + ?assertEqual(1, count_connections_in(Config, Username)), + ?assertEqual(5, count_channels_in(Config, Username)), + ?assertEqual(1, tracked_user_connection_count(Config, Username)), + ?assertEqual(5, tracked_user_channel_count(Config, Username)), + ?assertEqual(true, is_process_alive(Conn1)), + [?assertEqual(true, is_process_alive(Ch)) || Ch <- Chans1], + + [Conn2] = open_connections(Config, [{0, Username2}]), + Chans2 = [_|_] = open_channels(Conn2, 5), + ?assertEqual(1, count_connections_in(Config, Username2)), + ?assertEqual(5, count_channels_in(Config, Username2)), + ?assertEqual(1, tracked_user_connection_count(Config, Username2)), + ?assertEqual(5, tracked_user_channel_count(Config, Username2)), + ?assertEqual(true, is_process_alive(Conn2)), + [?assertEqual(true, is_process_alive(Ch)) || Ch <- Chans2], + + ?assertEqual(true, exists_in_tracked_connection_per_vhost_table(Config, Vhost)), + + rabbit_ct_broker_helpers:delete_vhost(Config, Vhost), + timer:sleep(200), + ?assertEqual(0, count_connections_in(Config, Username2)), + ?assertEqual(0, count_channels_in(Config, Username2)), + ?assertEqual(0, tracked_user_connection_count(Config, Username2)), + ?assertEqual(0, tracked_user_channel_count(Config, Username2)), + ?assertEqual(false, is_process_alive(Conn2)), + [?assertEqual(false, is_process_alive(Ch)) || Ch <- Chans2], + + ?assertEqual(0, count_connections_in(Config, Username)), + ?assertEqual(0, count_channels_in(Config, Username)), + ?assertEqual(0, tracked_user_connection_count(Config, Username)), + ?assertEqual(0, tracked_user_channel_count(Config, Username)), + ?assertEqual(false, is_process_alive(Conn1)), + [?assertEqual(false, is_process_alive(Ch)) || Ch <- Chans1], + + %% ensure vhost entry is cleared after 'tracking_execution_timeout' + ?assertEqual(false, exists_in_tracked_connection_per_vhost_table(Config, Vhost)), + + rabbit_ct_broker_helpers:add_vhost(Config, Vhost). + +single_node_vhost_down_mimic(Config) -> + Username = proplists:get_value(rmq_username, Config), + Username2 = <<"guest2">>, + + Vhost = proplists:get_value(rmq_vhost, Config), + + rabbit_ct_broker_helpers:add_user(Config, Username2), + rabbit_ct_broker_helpers:set_full_permissions(Config, Username2, Vhost), + + ?assertEqual(0, count_connections_in(Config, Username)), + ?assertEqual(0, count_connections_in(Config, Username2)), + ?assertEqual(0, count_channels_in(Config, Username)), + ?assertEqual(0, count_channels_in(Config, Username2)), + ?assertEqual(0, tracked_user_connection_count(Config, Username)), + ?assertEqual(0, tracked_user_connection_count(Config, Username2)), + ?assertEqual(0, tracked_user_channel_count(Config, Username)), + ?assertEqual(0, tracked_user_channel_count(Config, Username2)), + + [Conn1] = open_connections(Config, [0]), + Chans1 = [_|_] = open_channels(Conn1, 5), + ?assertEqual(1, count_connections_in(Config, Username)), + ?assertEqual(5, count_channels_in(Config, Username)), + ?assertEqual(1, tracked_user_connection_count(Config, Username)), + ?assertEqual(5, tracked_user_channel_count(Config, Username)), + ?assertEqual(true, is_process_alive(Conn1)), + [?assertEqual(true, is_process_alive(Ch)) || Ch <- Chans1], + + [Conn2] = open_connections(Config, [{0, Username2}]), + Chans2 = [_|_] = open_channels(Conn2, 5), + ?assertEqual(1, count_connections_in(Config, Username2)), + ?assertEqual(5, count_channels_in(Config, Username2)), + ?assertEqual(1, tracked_user_connection_count(Config, Username2)), + ?assertEqual(5, tracked_user_channel_count(Config, Username2)), + ?assertEqual(true, is_process_alive(Conn2)), + [?assertEqual(true, is_process_alive(Ch)) || Ch <- Chans2], + + %% mimic vhost down event, while connections exist + mimic_vhost_down(Config, 0, Vhost), + timer:sleep(200), + ?assertEqual(0, count_connections_in(Config, Username2)), + ?assertEqual(0, count_channels_in(Config, Username2)), + ?assertEqual(0, tracked_user_connection_count(Config, Username2)), + ?assertEqual(0, tracked_user_channel_count(Config, Username2)), + ?assertEqual(false, is_process_alive(Conn2)), + [?assertEqual(false, is_process_alive(Ch)) || Ch <- Chans2], + + ?assertEqual(0, count_connections_in(Config, Username)), + ?assertEqual(0, count_channels_in(Config, Username)), + ?assertEqual(0, tracked_user_connection_count(Config, Username)), + ?assertEqual(0, tracked_user_channel_count(Config, Username)), + ?assertEqual(false, is_process_alive(Conn1)), + [?assertEqual(false, is_process_alive(Ch)) || Ch <- Chans1]. + +cluster_user_deletion(Config) -> + set_tracking_execution_timeout(Config, 0, 100), + set_tracking_execution_timeout(Config, 1, 100), + Username = proplists:get_value(rmq_username, Config), + Username2 = <<"guest2">>, + + Vhost = proplists:get_value(rmq_vhost, Config), + + rabbit_ct_broker_helpers:add_user(Config, Username2), + rabbit_ct_broker_helpers:set_full_permissions(Config, Username2, Vhost), + + ?assertEqual(100, get_tracking_execution_timeout(Config, 0)), + ?assertEqual(100, get_tracking_execution_timeout(Config, 1)), + + ?assertEqual(0, count_connections_in(Config, Username)), + ?assertEqual(0, count_connections_in(Config, Username2)), + ?assertEqual(0, count_channels_in(Config, Username)), + ?assertEqual(0, count_channels_in(Config, Username2)), + ?assertEqual(0, tracked_user_connection_count(Config, Username)), + ?assertEqual(0, tracked_user_connection_count(Config, Username2)), + ?assertEqual(0, tracked_user_channel_count(Config, Username)), + ?assertEqual(0, tracked_user_channel_count(Config, Username2)), + + [Conn1] = open_connections(Config, [0]), + Chans1 = [_|_] = open_channels(Conn1, 5), + ?assertEqual(1, count_connections_in(Config, Username)), + ?assertEqual(5, count_channels_in(Config, Username)), + ?assertEqual(1, tracked_user_connection_count(Config, Username)), + ?assertEqual(5, tracked_user_channel_count(Config, Username)), + ?assertEqual(true, is_process_alive(Conn1)), + [?assertEqual(true, is_process_alive(Ch)) || Ch <- Chans1], + + [Conn2] = open_connections(Config, [{1, Username2}]), + Chans2 = [_|_] = open_channels(Conn2, 5), + ?assertEqual(1, count_connections_in(Config, Username2)), + ?assertEqual(5, count_channels_in(Config, Username2)), + ?assertEqual(1, tracked_user_connection_count(Config, Username2)), + ?assertEqual(5, tracked_user_channel_count(Config, Username2)), + ?assertEqual(true, is_process_alive(Conn2)), + [?assertEqual(true, is_process_alive(Ch)) || Ch <- Chans2], + + ?assertEqual(true, exists_in_tracked_connection_per_user_table(Config, 1, Username2)), + ?assertEqual(true, exists_in_tracked_channel_per_user_table(Config, 1, Username2)), + + rabbit_ct_broker_helpers:delete_user(Config, Username2), + timer:sleep(200), + ?assertEqual(0, count_connections_in(Config, Username2)), + ?assertEqual(0, count_channels_in(Config, Username2)), + ?assertEqual(0, tracked_user_connection_count(Config, Username2)), + ?assertEqual(0, tracked_user_channel_count(Config, Username2)), + ?assertEqual(false, is_process_alive(Conn2)), + [?assertEqual(false, is_process_alive(Ch)) || Ch <- Chans2], + + %% ensure user entry is cleared after 'tracking_execution_timeout' + ?assertEqual(false, exists_in_tracked_connection_per_user_table(Config, 1, Username2)), + ?assertEqual(false, exists_in_tracked_channel_per_user_table(Config, 1, Username2)), + + close_channels(Chans1), + ?awaitMatch(0, count_channels_in(Config, Username), 20000), + ?awaitMatch(0, tracked_user_channel_count(Config, Username), 20000), + + close_connections([Conn1]), + ?awaitMatch(0, count_connections_in(Config, Username), 20000), + ?awaitMatch(0, tracked_user_connection_count(Config, Username), 20000). + +cluster_vhost_deletion(Config) -> + set_tracking_execution_timeout(Config, 0, 100), + set_tracking_execution_timeout(Config, 1, 100), + Username = proplists:get_value(rmq_username, Config), + Username2 = <<"guest2">>, + + Vhost = proplists:get_value(rmq_vhost, Config), + + rabbit_ct_broker_helpers:add_user(Config, Username2), + rabbit_ct_broker_helpers:set_full_permissions(Config, Username2, Vhost), + + ?assertEqual(100, get_tracking_execution_timeout(Config, 0)), + ?assertEqual(100, get_tracking_execution_timeout(Config, 1)), + + ?assertEqual(0, count_connections_in(Config, Username)), + ?assertEqual(0, count_connections_in(Config, Username2)), + ?assertEqual(0, count_channels_in(Config, Username)), + ?assertEqual(0, count_channels_in(Config, Username2)), + ?assertEqual(0, tracked_user_connection_count(Config, Username)), + ?assertEqual(0, tracked_user_connection_count(Config, Username2)), + ?assertEqual(0, tracked_user_channel_count(Config, Username)), + ?assertEqual(0, tracked_user_channel_count(Config, Username2)), + + [Conn1] = open_connections(Config, [{0, Username}]), + Chans1 = [_|_] = open_channels(Conn1, 5), + ?assertEqual(1, count_connections_in(Config, Username)), + ?assertEqual(5, count_channels_in(Config, Username)), + ?assertEqual(1, tracked_user_connection_count(Config, Username)), + ?assertEqual(5, tracked_user_channel_count(Config, Username)), + ?assertEqual(true, is_process_alive(Conn1)), + [?assertEqual(true, is_process_alive(Ch)) || Ch <- Chans1], + + [Conn2] = open_connections(Config, [{1, Username2}]), + Chans2 = [_|_] = open_channels(Conn2, 5), + ?assertEqual(1, count_connections_in(Config, Username2)), + ?assertEqual(5, count_channels_in(Config, Username2)), + ?assertEqual(1, tracked_user_connection_count(Config, Username2)), + ?assertEqual(5, tracked_user_channel_count(Config, Username2)), + ?assertEqual(true, is_process_alive(Conn2)), + [?assertEqual(true, is_process_alive(Ch)) || Ch <- Chans2], + + ?assertEqual(true, exists_in_tracked_connection_per_vhost_table(Config, 0, Vhost)), + ?assertEqual(true, exists_in_tracked_connection_per_vhost_table(Config, 1, Vhost)), + + rabbit_ct_broker_helpers:delete_vhost(Config, Vhost), + timer:sleep(200), + ?assertEqual(0, count_connections_in(Config, Username2)), + ?assertEqual(0, count_channels_in(Config, Username2)), + ?assertEqual(0, tracked_user_connection_count(Config, Username2)), + ?assertEqual(0, tracked_user_channel_count(Config, Username2)), + ?assertEqual(false, is_process_alive(Conn2)), + [?assertEqual(false, is_process_alive(Ch)) || Ch <- Chans2], + + ?assertEqual(0, count_connections_in(Config, Username)), + ?assertEqual(0, count_channels_in(Config, Username)), + ?assertEqual(0, tracked_user_connection_count(Config, Username)), + ?assertEqual(0, tracked_user_channel_count(Config, Username)), + ?assertEqual(false, is_process_alive(Conn1)), + [?assertEqual(false, is_process_alive(Ch)) || Ch <- Chans1], + + %% ensure vhost entry is cleared after 'tracking_execution_timeout' + ?assertEqual(false, exists_in_tracked_connection_per_vhost_table(Config, 0, Vhost)), + ?assertEqual(false, exists_in_tracked_connection_per_vhost_table(Config, 1, Vhost)), + + rabbit_ct_broker_helpers:add_vhost(Config, Vhost), + rabbit_ct_broker_helpers:add_user(Config, Username), + rabbit_ct_broker_helpers:set_full_permissions(Config, Username, Vhost). + +cluster_vhost_down_mimic(Config) -> + Username = proplists:get_value(rmq_username, Config), + Username2 = <<"guest2">>, + + Vhost = proplists:get_value(rmq_vhost, Config), + + rabbit_ct_broker_helpers:add_user(Config, Username2), + rabbit_ct_broker_helpers:set_full_permissions(Config, Username2, Vhost), + + ?assertEqual(0, count_connections_in(Config, Username)), + ?assertEqual(0, count_connections_in(Config, Username2)), + ?assertEqual(0, count_channels_in(Config, Username)), + ?assertEqual(0, count_channels_in(Config, Username2)), + ?assertEqual(0, tracked_user_connection_count(Config, Username)), + ?assertEqual(0, tracked_user_connection_count(Config, Username2)), + ?assertEqual(0, tracked_user_channel_count(Config, Username)), + ?assertEqual(0, tracked_user_channel_count(Config, Username2)), + + [Conn1] = open_connections(Config, [{0, Username}]), + Chans1 = [_|_] = open_channels(Conn1, 5), + ?assertEqual(1, count_connections_in(Config, Username)), + ?assertEqual(5, count_channels_in(Config, Username)), + ?assertEqual(1, tracked_user_connection_count(Config, Username)), + ?assertEqual(5, tracked_user_channel_count(Config, Username)), + ?assertEqual(true, is_process_alive(Conn1)), + [?assertEqual(true, is_process_alive(Ch)) || Ch <- Chans1], + + [Conn2] = open_connections(Config, [{1, Username2}]), + Chans2 = [_|_] = open_channels(Conn2, 5), + ?assertEqual(1, count_connections_in(Config, Username2)), + ?assertEqual(5, count_channels_in(Config, Username2)), + ?assertEqual(1, tracked_user_connection_count(Config, Username2)), + ?assertEqual(5, tracked_user_channel_count(Config, Username2)), + ?assertEqual(true, is_process_alive(Conn2)), + [?assertEqual(true, is_process_alive(Ch)) || Ch <- Chans2], + + mimic_vhost_down(Config, 1, Vhost), + timer:sleep(100), + ?assertEqual(0, count_connections_in(Config, Username2)), + ?assertEqual(0, count_channels_in(Config, Username2)), + ?assertEqual(0, tracked_user_connection_count(Config, Username2)), + ?assertEqual(0, tracked_user_channel_count(Config, Username2)), + ?assertEqual(false, is_process_alive(Conn2)), + [?assertEqual(false, is_process_alive(Ch)) || Ch <- Chans2], + + %% gen_event notifies local handlers. remote connections still active + ?assertEqual(1, count_connections_in(Config, Username)), + ?assertEqual(5, count_channels_in(Config, Username)), + ?assertEqual(1, tracked_user_connection_count(Config, Username)), + ?assertEqual(5, tracked_user_channel_count(Config, Username)), + ?assertEqual(true, is_process_alive(Conn1)), + [?assertEqual(true, is_process_alive(Ch)) || Ch <- Chans1], + + mimic_vhost_down(Config, 0, Vhost), + timer:sleep(100), + ?assertEqual(0, count_connections_in(Config, Username)), + ?assertEqual(0, count_channels_in(Config, Username)), + ?assertEqual(0, tracked_user_connection_count(Config, Username)), + ?assertEqual(0, tracked_user_channel_count(Config, Username)), + ?assertEqual(false, is_process_alive(Conn1)), + [?assertEqual(false, is_process_alive(Ch)) || Ch <- Chans1]. + +cluster_node_removed(Config) -> + Username = proplists:get_value(rmq_username, Config), + Username2 = <<"guest2">>, + + Vhost = proplists:get_value(rmq_vhost, Config), + + rabbit_ct_broker_helpers:add_user(Config, Username2), + rabbit_ct_broker_helpers:set_full_permissions(Config, Username2, Vhost), + + ?assertEqual(0, count_connections_in(Config, Username)), + ?assertEqual(0, count_connections_in(Config, Username2)), + ?assertEqual(0, count_channels_in(Config, Username)), + ?assertEqual(0, count_channels_in(Config, Username2)), + ?assertEqual(0, tracked_user_connection_count(Config, Username)), + ?assertEqual(0, tracked_user_connection_count(Config, Username2)), + ?assertEqual(0, tracked_user_channel_count(Config, Username)), + ?assertEqual(0, tracked_user_channel_count(Config, Username2)), + + [Conn1] = open_connections(Config, [{0, Username}]), + Chans1 = [_|_] = open_channels(Conn1, 5), + ?assertEqual(1, count_connections_in(Config, Username)), + ?assertEqual(5, count_channels_in(Config, Username)), + ?assertEqual(1, tracked_user_connection_count(Config, Username)), + ?assertEqual(5, tracked_user_channel_count(Config, Username)), + ?assertEqual(true, is_process_alive(Conn1)), + [?assertEqual(true, is_process_alive(Ch)) || Ch <- Chans1], + + [Conn2] = open_connections(Config, [{1, Username2}]), + Chans2 = [_|_] = open_channels(Conn2, 5), + ?assertEqual(1, count_connections_in(Config, Username2)), + ?assertEqual(5, count_channels_in(Config, Username2)), + ?assertEqual(1, tracked_user_connection_count(Config, Username2)), + ?assertEqual(5, tracked_user_channel_count(Config, Username2)), + ?assertEqual(true, is_process_alive(Conn2)), + [?assertEqual(true, is_process_alive(Ch)) || Ch <- Chans2], + + rabbit_ct_broker_helpers:stop_broker(Config, 1), + timer:sleep(200), + ?assertEqual(1, count_connections_in(Config, Username)), + ?assertEqual(5, count_channels_in(Config, Username)), + ?assertEqual(1, tracked_user_connection_count(Config, Username)), + ?assertEqual(5, tracked_user_channel_count(Config, Username)), + ?assertEqual(true, is_process_alive(Conn1)), + [?assertEqual(true, is_process_alive(Ch)) || Ch <- Chans1], + + rabbit_ct_broker_helpers:forget_cluster_node(Config, 0, 1), + timer:sleep(200), + NodeName = rabbit_ct_broker_helpers:get_node_config(Config, 1, nodename), + + DroppedConnTrackingTables = + rabbit_connection_tracking:get_all_tracked_connection_table_names_for_node(NodeName), + [?assertEqual( + {'EXIT', {aborted, {no_exists, Tab, all}}}, + catch mnesia:table_info(Tab, all)) || Tab <- DroppedConnTrackingTables], + + DroppedChTrackingTables = + rabbit_channel_tracking:get_all_tracked_channel_table_names_for_node(NodeName), + [?assertEqual( + {'EXIT', {aborted, {no_exists, Tab, all}}}, + catch mnesia:table_info(Tab, all)) || Tab <- DroppedChTrackingTables], + + ?assertEqual(false, is_process_alive(Conn2)), + [?assertEqual(false, is_process_alive(Ch)) || Ch <- Chans2], + + ?assertEqual(1, count_connections_in(Config, Username)), + ?assertEqual(5, count_channels_in(Config, Username)), + ?assertEqual(1, tracked_user_connection_count(Config, Username)), + ?assertEqual(5, tracked_user_channel_count(Config, Username)), + ?assertEqual(true, is_process_alive(Conn1)), + [?assertEqual(true, is_process_alive(Ch)) || Ch <- Chans1], + + close_channels(Chans1), + ?awaitMatch(0, count_channels_in(Config, Username), 20000), + ?awaitMatch(0, tracked_user_channel_count(Config, Username), 20000), + + close_connections([Conn1]), + ?awaitMatch(0, count_connections_in(Config, Username), 20000), + ?awaitMatch(0, tracked_user_connection_count(Config, Username), 20000). + +%% ------------------------------------------------------------------- +%% Helpers +%% ------------------------------------------------------------------- + +open_connections(Config, NodesAndUsers) -> + % Randomly select connection type + OpenConnectionFun = case ?config(connection_type, Config) of + network -> open_unmanaged_connection; + direct -> open_unmanaged_connection_direct + end, + Conns = lists:map(fun + ({Node, User}) -> + rabbit_ct_client_helpers:OpenConnectionFun(Config, Node, + User, User); + (Node) -> + rabbit_ct_client_helpers:OpenConnectionFun(Config, Node) + end, NodesAndUsers), + timer:sleep(500), + Conns. + +close_connections(Conns) -> + lists:foreach(fun + (Conn) -> + rabbit_ct_client_helpers:close_connection(Conn) + end, Conns), + timer:sleep(500). + +kill_connections(Conns) -> + lists:foreach(fun + (Conn) -> + (catch exit(Conn, please_terminate)) + end, Conns), + timer:sleep(500). + +open_channels(Conn, N) -> + [begin + {ok, Ch} = amqp_connection:open_channel(Conn), + Ch + end || _ <- lists:seq(1, N)]. + +close_channels(Channels = [_|_]) -> + [rabbit_ct_client_helpers:close_channel(Ch) || Ch <- Channels]. + +count_connections_in(Config, Username) -> + length(connections_in(Config, Username)). + +connections_in(Config, Username) -> + connections_in(Config, 0, Username). +connections_in(Config, NodeIndex, Username) -> + tracked_list_of_user(Config, NodeIndex, rabbit_connection_tracking, Username). + +count_channels_in(Config, Username) -> + Channels = channels_in(Config, Username), + length([Ch || Ch = #tracked_channel{username = Username0} <- Channels, + Username =:= Username0]). + +channels_in(Config, Username) -> + channels_in(Config, 0, Username). +channels_in(Config, NodeIndex, Username) -> + tracked_list_of_user(Config, NodeIndex, rabbit_channel_tracking, Username). + +tracked_list_of_user(Config, NodeIndex, TrackingMod, Username) -> + rabbit_ct_broker_helpers:rpc(Config, NodeIndex, + TrackingMod, + list_of_user, [Username]). + +tracked_user_connection_count(Config, Username) -> + tracked_user_connection_count(Config, 0, Username). +tracked_user_connection_count(Config, NodeIndex, Username) -> + count_user_tracked_items(Config, NodeIndex, rabbit_connection_tracking, Username). + +tracked_user_channel_count(Config, Username) -> + tracked_user_channel_count(Config, 0, Username). +tracked_user_channel_count(Config, NodeIndex, Username) -> + count_user_tracked_items(Config, NodeIndex, rabbit_channel_tracking, Username). + +count_user_tracked_items(Config, NodeIndex, TrackingMod, Username) -> + rabbit_ct_broker_helpers:rpc(Config, NodeIndex, + TrackingMod, + count_tracked_items_in, [{user, Username}]). + +exists_in_tracked_connection_per_vhost_table(Config, VHost) -> + exists_in_tracked_connection_per_vhost_table(Config, 0, VHost). +exists_in_tracked_connection_per_vhost_table(Config, NodeIndex, VHost) -> + exists_in_tracking_table(Config, NodeIndex, + fun rabbit_connection_tracking:tracked_connection_per_vhost_table_name_for/1, + VHost). + +exists_in_tracked_connection_per_user_table(Config, Username) -> + exists_in_tracked_connection_per_user_table(Config, 0, Username). +exists_in_tracked_connection_per_user_table(Config, NodeIndex, Username) -> + exists_in_tracking_table(Config, NodeIndex, + fun rabbit_connection_tracking:tracked_connection_per_user_table_name_for/1, + Username). + +exists_in_tracked_channel_per_user_table(Config, Username) -> + exists_in_tracked_channel_per_user_table(Config, 0, Username). +exists_in_tracked_channel_per_user_table(Config, NodeIndex, Username) -> + exists_in_tracking_table(Config, NodeIndex, + fun rabbit_channel_tracking:tracked_channel_per_user_table_name_for/1, + Username). + +exists_in_tracking_table(Config, NodeIndex, TableNameFun, Key) -> + Node = rabbit_ct_broker_helpers:get_node_config( + Config, NodeIndex, nodename), + Tab = TableNameFun(Node), + AllKeys = rabbit_ct_broker_helpers:rpc(Config, NodeIndex, + mnesia, + dirty_all_keys, [Tab]), + lists:member(Key, AllKeys). + +mimic_vhost_down(Config, NodeIndex, VHost) -> + rabbit_ct_broker_helpers:rpc(Config, NodeIndex, + rabbit_vhost, vhost_down, [VHost]). + +all_connections(Config) -> + all_connections(Config, 0). +all_connections(Config, NodeIndex) -> + all_tracked_items(Config, NodeIndex, rabbit_connection_tracking). + +all_channels(Config) -> + all_channels(Config, 0). +all_channels(Config, NodeIndex) -> + all_tracked_items(Config, NodeIndex, rabbit_channel_tracking). + +all_tracked_items(Config, NodeIndex, TrackingMod) -> + rabbit_ct_broker_helpers:rpc(Config, NodeIndex, + TrackingMod, + list, []). + +set_up_vhost(Config, VHost) -> + rabbit_ct_broker_helpers:add_vhost(Config, VHost), + rabbit_ct_broker_helpers:set_full_permissions(Config, <<"guest">>, VHost), + set_vhost_connection_limit(Config, VHost, -1). + +set_vhost_connection_limit(Config, VHost, Count) -> + set_vhost_connection_limit(Config, 0, VHost, Count). + +set_vhost_connection_limit(Config, NodeIndex, VHost, Count) -> + Node = rabbit_ct_broker_helpers:get_node_config( + Config, NodeIndex, nodename), + ok = rabbit_ct_broker_helpers:control_action( + set_vhost_limits, Node, + ["{\"max-connections\": " ++ integer_to_list(Count) ++ "}"], + [{"-p", binary_to_list(VHost)}]). + +set_tracking_execution_timeout(Config, Timeout) -> + set_tracking_execution_timeout(Config, 0, Timeout). +set_tracking_execution_timeout(Config, NodeIndex, Timeout) -> + rabbit_ct_broker_helpers:rpc(Config, NodeIndex, + application, set_env, + [rabbit, tracking_execution_timeout, Timeout]). + +get_tracking_execution_timeout(Config) -> + get_tracking_execution_timeout(Config, 0). +get_tracking_execution_timeout(Config, NodeIndex) -> + {ok, Timeout} = rabbit_ct_broker_helpers:rpc( + Config, NodeIndex, + application, get_env, + [rabbit, tracking_execution_timeout]), + Timeout. + +await_running_node_refresh(_Config, _NodeIndex) -> + timer:sleep(250). + +expect_that_client_connection_is_rejected(Config) -> + expect_that_client_connection_is_rejected(Config, 0). + +expect_that_client_connection_is_rejected(Config, NodeIndex) -> + {error, not_allowed} = + rabbit_ct_client_helpers:open_unmanaged_connection(Config, NodeIndex). + +expect_that_client_connection_is_rejected(Config, NodeIndex, VHost) -> + {error, not_allowed} = + rabbit_ct_client_helpers:open_unmanaged_connection(Config, NodeIndex, VHost). diff --git a/deps/rabbit/test/per_user_connection_tracking_SUITE.erl b/deps/rabbit/test/per_user_connection_tracking_SUITE.erl new file mode 100644 index 0000000000..36b0962eac --- /dev/null +++ b/deps/rabbit/test/per_user_connection_tracking_SUITE.erl @@ -0,0 +1,269 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2011-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(per_user_connection_tracking_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-compile(export_all). + +all() -> + [ + {group, cluster_size_1_network}, + {group, cluster_size_2_network}, + {group, cluster_size_1_direct}, + {group, cluster_size_2_direct} + ]. + +groups() -> + ClusterSize1Tests = [ + single_node_list_of_user, + single_node_user_deletion_forces_connection_closure + ], + ClusterSize2Tests = [ + cluster_user_deletion_forces_connection_closure + ], + [ + {cluster_size_1_network, [], ClusterSize1Tests}, + {cluster_size_2_network, [], ClusterSize2Tests}, + {cluster_size_1_direct, [], ClusterSize1Tests}, + {cluster_size_2_direct, [], ClusterSize2Tests} + ]. + +suite() -> + [ + %% If a test hangs, no need to wait for 30 minutes. + {timetrap, {minutes, 8}} + ]. + +%% ------------------------------------------------------------------- +%% Testsuite setup/teardown. +%% ------------------------------------------------------------------- + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(cluster_size_1_network, Config) -> + Config1 = rabbit_ct_helpers:set_config(Config, [{connection_type, network}]), + init_per_multinode_group(cluster_size_1_network, Config1, 1); +init_per_group(cluster_size_2_network, Config) -> + Config1 = rabbit_ct_helpers:set_config(Config, [{connection_type, network}]), + init_per_multinode_group(cluster_size_2_network, Config1, 2); +init_per_group(cluster_size_1_direct, Config) -> + Config1 = rabbit_ct_helpers:set_config(Config, [{connection_type, direct}]), + init_per_multinode_group(cluster_size_1_direct, Config1, 1); +init_per_group(cluster_size_2_direct, Config) -> + Config1 = rabbit_ct_helpers:set_config(Config, [{connection_type, direct}]), + init_per_multinode_group(cluster_size_2_direct, Config1, 2). + +init_per_multinode_group(_Group, Config, NodeCount) -> + Suffix = rabbit_ct_helpers:testcase_absname(Config, "", "-"), + Config1 = rabbit_ct_helpers:set_config(Config, [ + {rmq_nodes_count, NodeCount}, + {rmq_nodename_suffix, Suffix} + ]), + rabbit_ct_helpers:run_steps(Config1, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()). + +end_per_group(_Group, Config) -> + rabbit_ct_helpers:run_steps(Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()). + +init_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_started(Config, Testcase), + clear_all_connection_tracking_tables(Config), + Config. + +end_per_testcase(Testcase, Config) -> + clear_all_connection_tracking_tables(Config), + rabbit_ct_helpers:testcase_finished(Config, Testcase). + +clear_all_connection_tracking_tables(Config) -> + [rabbit_ct_broker_helpers:rpc(Config, + N, + rabbit_connection_tracking, + clear_tracked_connection_tables_for_this_node, + []) || N <- rabbit_ct_broker_helpers:get_node_configs(Config, nodename)]. + +%% ------------------------------------------------------------------- +%% Test cases. +%% ------------------------------------------------------------------- +single_node_list_of_user(Config) -> + Username = proplists:get_value(rmq_username, Config), + Username2 = <<"guest2">>, + + Vhost = proplists:get_value(rmq_vhost, Config), + + rabbit_ct_broker_helpers:add_user(Config, Username2), + rabbit_ct_broker_helpers:set_full_permissions(Config, Username2, Vhost), + + ?assertEqual(0, length(connections_in(Config, Username))), + ?assertEqual(0, length(connections_in(Config, Username2))), + + [Conn1] = open_connections(Config, [0]), + [#tracked_connection{username = Username}] = connections_in(Config, Username), + close_connections([Conn1]), + ?assertEqual(0, length(connections_in(Config, Username))), + + [Conn2] = open_connections(Config, [{0, Username2}]), + [#tracked_connection{username = Username2}] = connections_in(Config, Username2), + + [Conn3] = open_connections(Config, [0]), + [#tracked_connection{username = Username}] = connections_in(Config, Username), + + [Conn4] = open_connections(Config, [0]), + kill_connections([Conn4]), + [#tracked_connection{username = Username}] = connections_in(Config, Username), + + [Conn5] = open_connections(Config, [0]), + [Username, Username] = + lists:map(fun (#tracked_connection{username = U}) -> U end, + connections_in(Config, Username)), + + close_connections([Conn2, Conn3, Conn5]), + rabbit_ct_broker_helpers:delete_user(Config, Username2), + ?assertEqual(0, length(all_connections(Config))). + +single_node_user_deletion_forces_connection_closure(Config) -> + Username = proplists:get_value(rmq_username, Config), + Username2 = <<"guest2">>, + + Vhost = proplists:get_value(rmq_vhost, Config), + + rabbit_ct_broker_helpers:add_user(Config, Username2), + rabbit_ct_broker_helpers:set_full_permissions(Config, Username2, Vhost), + + ?assertEqual(0, count_connections_in(Config, Username)), + ?assertEqual(0, count_connections_in(Config, Username2)), + + [Conn1] = open_connections(Config, [0]), + ?assertEqual(1, count_connections_in(Config, Username)), + + [_Conn2] = open_connections(Config, [{0, Username2}]), + ?assertEqual(1, count_connections_in(Config, Username2)), + + rabbit_ct_broker_helpers:delete_user(Config, Username2), + timer:sleep(200), + ?assertEqual(0, count_connections_in(Config, Username2)), + + close_connections([Conn1]), + ?assertEqual(0, count_connections_in(Config, Username)). + +cluster_user_deletion_forces_connection_closure(Config) -> + Username = proplists:get_value(rmq_username, Config), + Username2 = <<"guest2">>, + + Vhost = proplists:get_value(rmq_vhost, Config), + + rabbit_ct_broker_helpers:add_user(Config, Username2), + rabbit_ct_broker_helpers:set_full_permissions(Config, Username2, Vhost), + + ?assertEqual(0, count_connections_in(Config, Username)), + ?assertEqual(0, count_connections_in(Config, Username2)), + + [Conn1] = open_connections(Config, [{0, Username}]), + ?assertEqual(1, count_connections_in(Config, Username)), + + [_Conn2] = open_connections(Config, [{1, Username2}]), + ?assertEqual(1, count_connections_in(Config, Username2)), + + rabbit_ct_broker_helpers:delete_user(Config, Username2), + timer:sleep(200), + ?assertEqual(0, count_connections_in(Config, Username2)), + + close_connections([Conn1]), + ?assertEqual(0, count_connections_in(Config, Username)). + +%% ------------------------------------------------------------------- +%% Helpers +%% ------------------------------------------------------------------- + +open_connections(Config, NodesAndUsers) -> + % Randomly select connection type + OpenConnectionFun = case ?config(connection_type, Config) of + network -> open_unmanaged_connection; + direct -> open_unmanaged_connection_direct + end, + Conns = lists:map(fun + ({Node, User}) -> + rabbit_ct_client_helpers:OpenConnectionFun(Config, Node, + User, User); + (Node) -> + rabbit_ct_client_helpers:OpenConnectionFun(Config, Node) + end, NodesAndUsers), + timer:sleep(500), + Conns. + +close_connections(Conns) -> + lists:foreach(fun + (Conn) -> + rabbit_ct_client_helpers:close_connection(Conn) + end, Conns), + timer:sleep(500). + +kill_connections(Conns) -> + lists:foreach(fun + (Conn) -> + (catch exit(Conn, please_terminate)) + end, Conns), + timer:sleep(500). + + +count_connections_in(Config, Username) -> + length(connections_in(Config, Username)). + +connections_in(Config, Username) -> + connections_in(Config, 0, Username). +connections_in(Config, NodeIndex, Username) -> + rabbit_ct_broker_helpers:rpc(Config, NodeIndex, + rabbit_connection_tracking, + list_of_user, [Username]). + +all_connections(Config) -> + all_connections(Config, 0). +all_connections(Config, NodeIndex) -> + rabbit_ct_broker_helpers:rpc(Config, NodeIndex, + rabbit_connection_tracking, + list, []). + +set_up_vhost(Config, VHost) -> + rabbit_ct_broker_helpers:add_vhost(Config, VHost), + rabbit_ct_broker_helpers:set_full_permissions(Config, <<"guest">>, VHost), + set_vhost_connection_limit(Config, VHost, -1). + +set_vhost_connection_limit(Config, VHost, Count) -> + set_vhost_connection_limit(Config, 0, VHost, Count). + +set_vhost_connection_limit(Config, NodeIndex, VHost, Count) -> + Node = rabbit_ct_broker_helpers:get_node_config( + Config, NodeIndex, nodename), + ok = rabbit_ct_broker_helpers:control_action( + set_vhost_limits, Node, + ["{\"max-connections\": " ++ integer_to_list(Count) ++ "}"], + [{"-p", binary_to_list(VHost)}]). + +await_running_node_refresh(_Config, _NodeIndex) -> + timer:sleep(250). + +expect_that_client_connection_is_rejected(Config) -> + expect_that_client_connection_is_rejected(Config, 0). + +expect_that_client_connection_is_rejected(Config, NodeIndex) -> + {error, not_allowed} = + rabbit_ct_client_helpers:open_unmanaged_connection(Config, NodeIndex). + +expect_that_client_connection_is_rejected(Config, NodeIndex, VHost) -> + {error, not_allowed} = + rabbit_ct_client_helpers:open_unmanaged_connection(Config, NodeIndex, VHost). diff --git a/deps/rabbit/test/per_vhost_connection_limit_SUITE.erl b/deps/rabbit/test/per_vhost_connection_limit_SUITE.erl new file mode 100644 index 0000000000..a140b3e829 --- /dev/null +++ b/deps/rabbit/test/per_vhost_connection_limit_SUITE.erl @@ -0,0 +1,751 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2011-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(per_vhost_connection_limit_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-compile(export_all). + +all() -> + [ + {group, cluster_size_1_network}, + {group, cluster_size_2_network}, + {group, cluster_size_1_direct}, + {group, cluster_size_2_direct} + ]. + +groups() -> + ClusterSize1Tests = [ + most_basic_single_node_connection_count, + single_node_single_vhost_connection_count, + single_node_multiple_vhosts_connection_count, + single_node_list_in_vhost, + single_node_single_vhost_limit, + single_node_single_vhost_zero_limit, + single_node_multiple_vhosts_limit, + single_node_multiple_vhosts_zero_limit + ], + ClusterSize2Tests = [ + most_basic_cluster_connection_count, + cluster_single_vhost_connection_count, + cluster_multiple_vhosts_connection_count, + cluster_node_restart_connection_count, + cluster_node_list_on_node, + cluster_single_vhost_limit, + cluster_single_vhost_limit2, + cluster_single_vhost_zero_limit, + cluster_multiple_vhosts_zero_limit + ], + [ + {cluster_size_1_network, [], ClusterSize1Tests}, + {cluster_size_2_network, [], ClusterSize2Tests}, + {cluster_size_1_direct, [], ClusterSize1Tests}, + {cluster_size_2_direct, [], ClusterSize2Tests}, + {cluster_rename, [], [ + vhost_limit_after_node_renamed + ]} + ]. + +suite() -> + [ + %% If a test hangs, no need to wait for 30 minutes. + {timetrap, {minutes, 8}} + ]. + +%% see partitions_SUITE +-define(DELAY, 9000). + +%% ------------------------------------------------------------------- +%% Testsuite setup/teardown. +%% ------------------------------------------------------------------- + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(cluster_size_1_network, Config) -> + Config1 = rabbit_ct_helpers:set_config(Config, [{connection_type, network}]), + init_per_multinode_group(cluster_size_1_network, Config1, 1); +init_per_group(cluster_size_2_network, Config) -> + Config1 = rabbit_ct_helpers:set_config(Config, [{connection_type, network}]), + init_per_multinode_group(cluster_size_2_network, Config1, 2); +init_per_group(cluster_size_1_direct, Config) -> + Config1 = rabbit_ct_helpers:set_config(Config, [{connection_type, direct}]), + init_per_multinode_group(cluster_size_1_direct, Config1, 1); +init_per_group(cluster_size_2_direct, Config) -> + Config1 = rabbit_ct_helpers:set_config(Config, [{connection_type, direct}]), + init_per_multinode_group(cluster_size_2_direct, Config1, 2); + +init_per_group(cluster_rename, Config) -> + init_per_multinode_group(cluster_rename, Config, 2). + +init_per_multinode_group(Group, Config, NodeCount) -> + Suffix = rabbit_ct_helpers:testcase_absname(Config, "", "-"), + Config1 = rabbit_ct_helpers:set_config(Config, [ + {rmq_nodes_count, NodeCount}, + {rmq_nodename_suffix, Suffix} + ]), + case Group of + cluster_rename -> + % The broker is managed by {init,end}_per_testcase(). + Config1; + _ -> + rabbit_ct_helpers:run_steps(Config1, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()) + end. + +end_per_group(cluster_rename, Config) -> + % The broker is managed by {init,end}_per_testcase(). + Config; +end_per_group(_Group, Config) -> + rabbit_ct_helpers:run_steps(Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()). + +init_per_testcase(vhost_limit_after_node_renamed = Testcase, Config) -> + rabbit_ct_helpers:testcase_started(Config, Testcase), + rabbit_ct_helpers:run_steps(Config, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()); +init_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_started(Config, Testcase), + clear_all_connection_tracking_tables(Config), + Config. + +end_per_testcase(vhost_limit_after_node_renamed = Testcase, Config) -> + Config1 = ?config(save_config, Config), + rabbit_ct_helpers:run_steps(Config1, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()), + rabbit_ct_helpers:testcase_finished(Config1, Testcase); +end_per_testcase(Testcase, Config) -> + clear_all_connection_tracking_tables(Config), + rabbit_ct_helpers:testcase_finished(Config, Testcase). + +clear_all_connection_tracking_tables(Config) -> + rabbit_ct_broker_helpers:rpc_all( + Config, + rabbit_connection_tracking, + clear_tracked_connection_tables_for_this_node, + []). + +%% ------------------------------------------------------------------- +%% Test cases. +%% ------------------------------------------------------------------- + +most_basic_single_node_connection_count(Config) -> + VHost = <<"/">>, + ?assertEqual(0, count_connections_in(Config, VHost)), + [Conn] = open_connections(Config, [0]), + ?assertEqual(1, count_connections_in(Config, VHost)), + close_connections([Conn]), + ?assertEqual(0, count_connections_in(Config, VHost)). + +single_node_single_vhost_connection_count(Config) -> + VHost = <<"/">>, + ?assertEqual(0, count_connections_in(Config, VHost)), + + [Conn1] = open_connections(Config, [0]), + ?assertEqual(1, count_connections_in(Config, VHost)), + close_connections([Conn1]), + ?assertEqual(0, count_connections_in(Config, VHost)), + + [Conn2] = open_connections(Config, [0]), + ?assertEqual(1, count_connections_in(Config, VHost)), + + [Conn3] = open_connections(Config, [0]), + ?assertEqual(2, count_connections_in(Config, VHost)), + + [Conn4] = open_connections(Config, [0]), + ?assertEqual(3, count_connections_in(Config, VHost)), + + kill_connections([Conn4]), + ?assertEqual(2, count_connections_in(Config, VHost)), + + [Conn5] = open_connections(Config, [0]), + ?assertEqual(3, count_connections_in(Config, VHost)), + + close_connections([Conn2, Conn3, Conn5]), + ?assertEqual(0, count_connections_in(Config, VHost)). + +single_node_multiple_vhosts_connection_count(Config) -> + VHost1 = <<"vhost1">>, + VHost2 = <<"vhost2">>, + + set_up_vhost(Config, VHost1), + set_up_vhost(Config, VHost2), + + ?assertEqual(0, count_connections_in(Config, VHost1)), + ?assertEqual(0, count_connections_in(Config, VHost2)), + + [Conn1] = open_connections(Config, [{0, VHost1}]), + ?assertEqual(1, count_connections_in(Config, VHost1)), + close_connections([Conn1]), + ?assertEqual(0, count_connections_in(Config, VHost1)), + + [Conn2] = open_connections(Config, [{0, VHost2}]), + ?assertEqual(1, count_connections_in(Config, VHost2)), + + [Conn3] = open_connections(Config, [{0, VHost1}]), + ?assertEqual(1, count_connections_in(Config, VHost1)), + ?assertEqual(1, count_connections_in(Config, VHost2)), + + [Conn4] = open_connections(Config, [{0, VHost1}]), + ?assertEqual(2, count_connections_in(Config, VHost1)), + + kill_connections([Conn4]), + ?assertEqual(1, count_connections_in(Config, VHost1)), + + [Conn5] = open_connections(Config, [{0, VHost2}]), + ?assertEqual(2, count_connections_in(Config, VHost2)), + + [Conn6] = open_connections(Config, [{0, VHost2}]), + ?assertEqual(3, count_connections_in(Config, VHost2)), + + close_connections([Conn2, Conn3, Conn5, Conn6]), + ?assertEqual(0, count_connections_in(Config, VHost1)), + ?assertEqual(0, count_connections_in(Config, VHost2)), + + rabbit_ct_broker_helpers:delete_vhost(Config, VHost1), + rabbit_ct_broker_helpers:delete_vhost(Config, VHost2). + +single_node_list_in_vhost(Config) -> + VHost1 = <<"vhost1">>, + VHost2 = <<"vhost2">>, + + set_up_vhost(Config, VHost1), + set_up_vhost(Config, VHost2), + + ?assertEqual(0, length(connections_in(Config, VHost1))), + ?assertEqual(0, length(connections_in(Config, VHost2))), + + [Conn1] = open_connections(Config, [{0, VHost1}]), + [#tracked_connection{vhost = VHost1}] = connections_in(Config, VHost1), + close_connections([Conn1]), + ?assertEqual(0, length(connections_in(Config, VHost1))), + + [Conn2] = open_connections(Config, [{0, VHost2}]), + [#tracked_connection{vhost = VHost2}] = connections_in(Config, VHost2), + + [Conn3] = open_connections(Config, [{0, VHost1}]), + [#tracked_connection{vhost = VHost1}] = connections_in(Config, VHost1), + + [Conn4] = open_connections(Config, [{0, VHost1}]), + kill_connections([Conn4]), + [#tracked_connection{vhost = VHost1}] = connections_in(Config, VHost1), + + [Conn5, Conn6] = open_connections(Config, [{0, VHost2}, {0, VHost2}]), + [<<"vhost1">>, <<"vhost2">>] = + lists:usort(lists:map(fun (#tracked_connection{vhost = V}) -> V end, + all_connections(Config))), + + close_connections([Conn2, Conn3, Conn5, Conn6]), + ?assertEqual(0, length(all_connections(Config))), + + rabbit_ct_broker_helpers:delete_vhost(Config, VHost1), + rabbit_ct_broker_helpers:delete_vhost(Config, VHost2). + +most_basic_cluster_connection_count(Config) -> + VHost = <<"/">>, + ?assertEqual(0, count_connections_in(Config, VHost)), + [Conn1] = open_connections(Config, [0]), + ?assertEqual(1, count_connections_in(Config, VHost)), + + [Conn2] = open_connections(Config, [1]), + ?assertEqual(2, count_connections_in(Config, VHost)), + + [Conn3] = open_connections(Config, [1]), + ?assertEqual(3, count_connections_in(Config, VHost)), + + close_connections([Conn1, Conn2, Conn3]), + ?assertEqual(0, count_connections_in(Config, VHost)). + +cluster_single_vhost_connection_count(Config) -> + VHost = <<"/">>, + ?assertEqual(0, count_connections_in(Config, VHost)), + + [Conn1] = open_connections(Config, [0]), + ?assertEqual(1, count_connections_in(Config, VHost)), + close_connections([Conn1]), + ?assertEqual(0, count_connections_in(Config, VHost)), + + [Conn2] = open_connections(Config, [1]), + ?assertEqual(1, count_connections_in(Config, VHost)), + + [Conn3] = open_connections(Config, [0]), + ?assertEqual(2, count_connections_in(Config, VHost)), + + [Conn4] = open_connections(Config, [1]), + ?assertEqual(3, count_connections_in(Config, VHost)), + + kill_connections([Conn4]), + ?assertEqual(2, count_connections_in(Config, VHost)), + + [Conn5] = open_connections(Config, [1]), + ?assertEqual(3, count_connections_in(Config, VHost)), + + close_connections([Conn2, Conn3, Conn5]), + ?assertEqual(0, count_connections_in(Config, VHost)). + +cluster_multiple_vhosts_connection_count(Config) -> + VHost1 = <<"vhost1">>, + VHost2 = <<"vhost2">>, + + set_up_vhost(Config, VHost1), + set_up_vhost(Config, VHost2), + + ?assertEqual(0, count_connections_in(Config, VHost1)), + ?assertEqual(0, count_connections_in(Config, VHost2)), + + [Conn1] = open_connections(Config, [{0, VHost1}]), + ?assertEqual(1, count_connections_in(Config, VHost1)), + close_connections([Conn1]), + ?assertEqual(0, count_connections_in(Config, VHost1)), + + [Conn2] = open_connections(Config, [{1, VHost2}]), + ?assertEqual(1, count_connections_in(Config, VHost2)), + + [Conn3] = open_connections(Config, [{1, VHost1}]), + ?assertEqual(1, count_connections_in(Config, VHost1)), + ?assertEqual(1, count_connections_in(Config, VHost2)), + + [Conn4] = open_connections(Config, [{0, VHost1}]), + ?assertEqual(2, count_connections_in(Config, VHost1)), + + kill_connections([Conn4]), + ?assertEqual(1, count_connections_in(Config, VHost1)), + + [Conn5] = open_connections(Config, [{1, VHost2}]), + ?assertEqual(2, count_connections_in(Config, VHost2)), + + [Conn6] = open_connections(Config, [{0, VHost2}]), + ?assertEqual(3, count_connections_in(Config, VHost2)), + + close_connections([Conn2, Conn3, Conn5, Conn6]), + ?assertEqual(0, count_connections_in(Config, VHost1)), + ?assertEqual(0, count_connections_in(Config, VHost2)), + + rabbit_ct_broker_helpers:delete_vhost(Config, VHost1), + rabbit_ct_broker_helpers:delete_vhost(Config, VHost2). + +cluster_node_restart_connection_count(Config) -> + VHost = <<"/">>, + ?assertEqual(0, count_connections_in(Config, VHost)), + + [Conn1] = open_connections(Config, [0]), + ?assertEqual(1, count_connections_in(Config, VHost)), + close_connections([Conn1]), + ?assertEqual(0, count_connections_in(Config, VHost)), + + [Conn2] = open_connections(Config, [1]), + ?assertEqual(1, count_connections_in(Config, VHost)), + + [Conn3] = open_connections(Config, [0]), + ?assertEqual(2, count_connections_in(Config, VHost)), + + [Conn4] = open_connections(Config, [1]), + ?assertEqual(3, count_connections_in(Config, VHost)), + + [Conn5] = open_connections(Config, [1]), + ?assertEqual(4, count_connections_in(Config, VHost)), + + rabbit_ct_broker_helpers:restart_broker(Config, 1), + ?assertEqual(1, count_connections_in(Config, VHost)), + + close_connections([Conn2, Conn3, Conn4, Conn5]), + ?assertEqual(0, count_connections_in(Config, VHost)). + +cluster_node_list_on_node(Config) -> + [A, B] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + ?assertEqual(0, length(all_connections(Config))), + ?assertEqual(0, length(connections_on_node(Config, 0))), + + [Conn1] = open_connections(Config, [0]), + [#tracked_connection{node = A}] = connections_on_node(Config, 0), + close_connections([Conn1]), + ?assertEqual(0, length(connections_on_node(Config, 0))), + + [_Conn2] = open_connections(Config, [1]), + [#tracked_connection{node = B}] = connections_on_node(Config, 1), + + [Conn3] = open_connections(Config, [0]), + ?assertEqual(1, length(connections_on_node(Config, 0))), + + [Conn4] = open_connections(Config, [1]), + ?assertEqual(2, length(connections_on_node(Config, 1))), + + kill_connections([Conn4]), + ?assertEqual(1, length(connections_on_node(Config, 1))), + + [Conn5] = open_connections(Config, [0]), + ?assertEqual(2, length(connections_on_node(Config, 0))), + + rabbit_ct_broker_helpers:stop_broker(Config, 1), + await_running_node_refresh(Config, 0), + + ?assertEqual(2, length(all_connections(Config))), + ?assertEqual(0, length(connections_on_node(Config, 0, B))), + + close_connections([Conn3, Conn5]), + ?assertEqual(0, length(all_connections(Config, 0))), + + rabbit_ct_broker_helpers:start_broker(Config, 1). + +single_node_single_vhost_limit(Config) -> + single_node_single_vhost_limit_with(Config, 5), + single_node_single_vhost_limit_with(Config, -1). + +single_node_single_vhost_limit_with(Config, WatermarkLimit) -> + VHost = <<"/">>, + set_vhost_connection_limit(Config, VHost, 3), + + ?assertEqual(0, count_connections_in(Config, VHost)), + + [Conn1, Conn2, Conn3] = open_connections(Config, [0, 0, 0]), + + %% we've crossed the limit + expect_that_client_connection_is_rejected(Config, 0), + expect_that_client_connection_is_rejected(Config, 0), + expect_that_client_connection_is_rejected(Config, 0), + + set_vhost_connection_limit(Config, VHost, WatermarkLimit), + [Conn4, Conn5] = open_connections(Config, [0, 0]), + + close_connections([Conn1, Conn2, Conn3, Conn4, Conn5]), + ?assertEqual(0, count_connections_in(Config, VHost)), + + set_vhost_connection_limit(Config, VHost, -1). + +single_node_single_vhost_zero_limit(Config) -> + VHost = <<"/">>, + set_vhost_connection_limit(Config, VHost, 0), + + ?assertEqual(0, count_connections_in(Config, VHost)), + + %% with limit = 0 no connections are allowed + expect_that_client_connection_is_rejected(Config), + expect_that_client_connection_is_rejected(Config), + expect_that_client_connection_is_rejected(Config), + + set_vhost_connection_limit(Config, VHost, -1), + [Conn1, Conn2] = open_connections(Config, [0, 0]), + + close_connections([Conn1, Conn2]), + ?assertEqual(0, count_connections_in(Config, VHost)). + + +single_node_multiple_vhosts_limit(Config) -> + VHost1 = <<"vhost1">>, + VHost2 = <<"vhost2">>, + + set_up_vhost(Config, VHost1), + set_up_vhost(Config, VHost2), + + set_vhost_connection_limit(Config, VHost1, 2), + set_vhost_connection_limit(Config, VHost2, 2), + + ?assertEqual(0, count_connections_in(Config, VHost1)), + ?assertEqual(0, count_connections_in(Config, VHost2)), + + [Conn1, Conn2, Conn3, Conn4] = open_connections(Config, [ + {0, VHost1}, + {0, VHost1}, + {0, VHost2}, + {0, VHost2}]), + + %% we've crossed the limit + expect_that_client_connection_is_rejected(Config, 0, VHost1), + expect_that_client_connection_is_rejected(Config, 0, VHost2), + + [Conn5] = open_connections(Config, [0]), + + set_vhost_connection_limit(Config, VHost1, 5), + set_vhost_connection_limit(Config, VHost2, -10), + + [Conn6, Conn7, Conn8, Conn9, Conn10] = open_connections(Config, [ + {0, VHost1}, + {0, VHost1}, + {0, VHost1}, + {0, VHost2}, + {0, VHost2}]), + + close_connections([Conn1, Conn2, Conn3, Conn4, Conn5, + Conn6, Conn7, Conn8, Conn9, Conn10]), + ?assertEqual(0, count_connections_in(Config, VHost1)), + ?assertEqual(0, count_connections_in(Config, VHost2)), + + set_vhost_connection_limit(Config, VHost1, -1), + set_vhost_connection_limit(Config, VHost2, -1), + + rabbit_ct_broker_helpers:delete_vhost(Config, VHost1), + rabbit_ct_broker_helpers:delete_vhost(Config, VHost2). + + +single_node_multiple_vhosts_zero_limit(Config) -> + VHost1 = <<"vhost1">>, + VHost2 = <<"vhost2">>, + + set_up_vhost(Config, VHost1), + set_up_vhost(Config, VHost2), + + set_vhost_connection_limit(Config, VHost1, 0), + set_vhost_connection_limit(Config, VHost2, 0), + + ?assertEqual(0, count_connections_in(Config, VHost1)), + ?assertEqual(0, count_connections_in(Config, VHost2)), + + %% with limit = 0 no connections are allowed + expect_that_client_connection_is_rejected(Config, 0, VHost1), + expect_that_client_connection_is_rejected(Config, 0, VHost2), + expect_that_client_connection_is_rejected(Config, 0, VHost1), + + set_vhost_connection_limit(Config, VHost1, -1), + [Conn1, Conn2] = open_connections(Config, [{0, VHost1}, {0, VHost1}]), + + close_connections([Conn1, Conn2]), + ?assertEqual(0, count_connections_in(Config, VHost1)), + ?assertEqual(0, count_connections_in(Config, VHost2)), + + set_vhost_connection_limit(Config, VHost1, -1), + set_vhost_connection_limit(Config, VHost2, -1). + + +cluster_single_vhost_limit(Config) -> + VHost = <<"/">>, + set_vhost_connection_limit(Config, VHost, 2), + + ?assertEqual(0, count_connections_in(Config, VHost)), + + %% here connections are opened to different nodes + [Conn1, Conn2] = open_connections(Config, [{0, VHost}, {1, VHost}]), + + %% we've crossed the limit + expect_that_client_connection_is_rejected(Config, 0, VHost), + expect_that_client_connection_is_rejected(Config, 1, VHost), + + set_vhost_connection_limit(Config, VHost, 5), + + [Conn3, Conn4] = open_connections(Config, [{0, VHost}, {0, VHost}]), + + close_connections([Conn1, Conn2, Conn3, Conn4]), + ?assertEqual(0, count_connections_in(Config, VHost)), + + set_vhost_connection_limit(Config, VHost, -1). + +cluster_single_vhost_limit2(Config) -> + VHost = <<"/">>, + set_vhost_connection_limit(Config, VHost, 2), + + ?assertEqual(0, count_connections_in(Config, VHost)), + + %% here a limit is reached on one node first + [Conn1, Conn2] = open_connections(Config, [{0, VHost}, {0, VHost}]), + + %% we've crossed the limit + expect_that_client_connection_is_rejected(Config, 0, VHost), + expect_that_client_connection_is_rejected(Config, 1, VHost), + + set_vhost_connection_limit(Config, VHost, 5), + + [Conn3, Conn4, Conn5, {error, not_allowed}] = open_connections(Config, [ + {1, VHost}, + {1, VHost}, + {1, VHost}, + {1, VHost}]), + + close_connections([Conn1, Conn2, Conn3, Conn4, Conn5]), + ?assertEqual(0, count_connections_in(Config, VHost)), + + set_vhost_connection_limit(Config, VHost, -1). + + +cluster_single_vhost_zero_limit(Config) -> + VHost = <<"/">>, + set_vhost_connection_limit(Config, VHost, 0), + + ?assertEqual(0, count_connections_in(Config, VHost)), + + %% with limit = 0 no connections are allowed + expect_that_client_connection_is_rejected(Config, 0), + expect_that_client_connection_is_rejected(Config, 1), + expect_that_client_connection_is_rejected(Config, 0), + + set_vhost_connection_limit(Config, VHost, -1), + [Conn1, Conn2, Conn3, Conn4] = open_connections(Config, [0, 1, 0, 1]), + + close_connections([Conn1, Conn2, Conn3, Conn4]), + ?assertEqual(0, count_connections_in(Config, VHost)), + + set_vhost_connection_limit(Config, VHost, -1). + + +cluster_multiple_vhosts_zero_limit(Config) -> + VHost1 = <<"vhost1">>, + VHost2 = <<"vhost2">>, + + set_up_vhost(Config, VHost1), + set_up_vhost(Config, VHost2), + + set_vhost_connection_limit(Config, VHost1, 0), + set_vhost_connection_limit(Config, VHost2, 0), + + ?assertEqual(0, count_connections_in(Config, VHost1)), + ?assertEqual(0, count_connections_in(Config, VHost2)), + + %% with limit = 0 no connections are allowed + expect_that_client_connection_is_rejected(Config, 0, VHost1), + expect_that_client_connection_is_rejected(Config, 0, VHost2), + expect_that_client_connection_is_rejected(Config, 1, VHost1), + expect_that_client_connection_is_rejected(Config, 1, VHost2), + + set_vhost_connection_limit(Config, VHost1, -1), + set_vhost_connection_limit(Config, VHost2, -1), + + [Conn1, Conn2, Conn3, Conn4] = open_connections(Config, [ + {0, VHost1}, + {0, VHost2}, + {1, VHost1}, + {1, VHost2}]), + + close_connections([Conn1, Conn2, Conn3, Conn4]), + ?assertEqual(0, count_connections_in(Config, VHost1)), + ?assertEqual(0, count_connections_in(Config, VHost2)), + + set_vhost_connection_limit(Config, VHost1, -1), + set_vhost_connection_limit(Config, VHost2, -1). + +vhost_limit_after_node_renamed(Config) -> + A = rabbit_ct_broker_helpers:get_node_config(Config, 0, nodename), + + VHost = <<"/renaming_node">>, + set_up_vhost(Config, VHost), + set_vhost_connection_limit(Config, VHost, 2), + + ?assertEqual(0, count_connections_in(Config, VHost)), + + [Conn1, Conn2, {error, not_allowed}] = open_connections(Config, + [{0, VHost}, {1, VHost}, {0, VHost}]), + ?assertEqual(2, count_connections_in(Config, VHost)), + close_connections([Conn1, Conn2]), + + Config1 = cluster_rename_SUITE:stop_rename_start(Config, A, [A, 'new-A']), + + ?assertEqual(0, count_connections_in(Config1, VHost)), + + [Conn3, Conn4, {error, not_allowed}] = open_connections(Config, + [{0, VHost}, {1, VHost}, {0, VHost}]), + ?assertEqual(2, count_connections_in(Config1, VHost)), + close_connections([Conn3, Conn4]), + + set_vhost_connection_limit(Config1, VHost, -1), + {save_config, Config1}. + +%% ------------------------------------------------------------------- +%% Helpers +%% ------------------------------------------------------------------- + +open_connections(Config, NodesAndVHosts) -> + % Randomly select connection type + OpenConnectionFun = case ?config(connection_type, Config) of + network -> open_unmanaged_connection; + direct -> open_unmanaged_connection_direct + end, + Conns = lists:map(fun + ({Node, VHost}) -> + rabbit_ct_client_helpers:OpenConnectionFun(Config, Node, + VHost); + (Node) -> + rabbit_ct_client_helpers:OpenConnectionFun(Config, Node) + end, NodesAndVHosts), + timer:sleep(500), + Conns. + +close_connections(Conns) -> + lists:foreach(fun + (Conn) -> + rabbit_ct_client_helpers:close_connection(Conn) + end, Conns), + timer:sleep(500). + +kill_connections(Conns) -> + lists:foreach(fun + (Conn) -> + (catch exit(Conn, please_terminate)) + end, Conns), + timer:sleep(500). + +count_connections_in(Config, VHost) -> + count_connections_in(Config, VHost, 0). +count_connections_in(Config, VHost, NodeIndex) -> + timer:sleep(200), + rabbit_ct_broker_helpers:rpc(Config, NodeIndex, + rabbit_connection_tracking, + count_tracked_items_in, [{vhost, VHost}]). + +connections_in(Config, VHost) -> + connections_in(Config, 0, VHost). +connections_in(Config, NodeIndex, VHost) -> + rabbit_ct_broker_helpers:rpc(Config, NodeIndex, + rabbit_connection_tracking, + list, [VHost]). + +connections_on_node(Config) -> + connections_on_node(Config, 0). +connections_on_node(Config, NodeIndex) -> + Node = rabbit_ct_broker_helpers:get_node_config(Config, NodeIndex, nodename), + rabbit_ct_broker_helpers:rpc(Config, NodeIndex, + rabbit_connection_tracking, + list_on_node, [Node]). +connections_on_node(Config, NodeIndex, NodeForListing) -> + rabbit_ct_broker_helpers:rpc(Config, NodeIndex, + rabbit_connection_tracking, + list_on_node, [NodeForListing]). + +all_connections(Config) -> + all_connections(Config, 0). +all_connections(Config, NodeIndex) -> + rabbit_ct_broker_helpers:rpc(Config, NodeIndex, + rabbit_connection_tracking, + list, []). + +set_up_vhost(Config, VHost) -> + rabbit_ct_broker_helpers:add_vhost(Config, VHost), + rabbit_ct_broker_helpers:set_full_permissions(Config, <<"guest">>, VHost), + set_vhost_connection_limit(Config, VHost, -1). + +set_vhost_connection_limit(Config, VHost, Count) -> + set_vhost_connection_limit(Config, 0, VHost, Count). + +set_vhost_connection_limit(Config, NodeIndex, VHost, Count) -> + Node = rabbit_ct_broker_helpers:get_node_config( + Config, NodeIndex, nodename), + ok = rabbit_ct_broker_helpers:control_action( + set_vhost_limits, Node, + ["{\"max-connections\": " ++ integer_to_list(Count) ++ "}"], + [{"-p", binary_to_list(VHost)}]). + +await_running_node_refresh(_Config, _NodeIndex) -> + timer:sleep(250). + +expect_that_client_connection_is_rejected(Config) -> + expect_that_client_connection_is_rejected(Config, 0). + +expect_that_client_connection_is_rejected(Config, NodeIndex) -> + {error, not_allowed} = + rabbit_ct_client_helpers:open_unmanaged_connection(Config, NodeIndex). + +expect_that_client_connection_is_rejected(Config, NodeIndex, VHost) -> + {error, not_allowed} = + rabbit_ct_client_helpers:open_unmanaged_connection(Config, NodeIndex, VHost). diff --git a/deps/rabbit/test/per_vhost_connection_limit_partitions_SUITE.erl b/deps/rabbit/test/per_vhost_connection_limit_partitions_SUITE.erl new file mode 100644 index 0000000000..2748d95592 --- /dev/null +++ b/deps/rabbit/test/per_vhost_connection_limit_partitions_SUITE.erl @@ -0,0 +1,150 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2011-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(per_vhost_connection_limit_partitions_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-compile(export_all). + +-import(rabbit_ct_client_helpers, [open_unmanaged_connection/2, + open_unmanaged_connection/3]). + + +all() -> + [ + {group, net_ticktime_1} + ]. + +groups() -> + [ + {net_ticktime_1, [], [ + cluster_full_partition_with_autoheal + ]} + ]. + +suite() -> + [ + %% If a test hangs, no need to wait for 30 minutes. + {timetrap, {minutes, 8}} + ]. + +%% see partitions_SUITE +-define(DELAY, 12000). + +%% ------------------------------------------------------------------- +%% Testsuite setup/teardown. +%% ------------------------------------------------------------------- + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config, [ + fun rabbit_ct_broker_helpers:configure_dist_proxy/1 + ]). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(net_ticktime_1 = Group, Config) -> + Config1 = rabbit_ct_helpers:set_config(Config, [{net_ticktime, 1}]), + init_per_multinode_group(Group, Config1, 3). + +init_per_multinode_group(_Group, Config, NodeCount) -> + Suffix = rabbit_ct_helpers:testcase_absname(Config, "", "-"), + Config1 = rabbit_ct_helpers:set_config(Config, [ + {rmq_nodes_count, NodeCount}, + {rmq_nodename_suffix, Suffix} + ]), + rabbit_ct_helpers:run_steps(Config1, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()). + +end_per_group(_Group, Config) -> + rabbit_ct_helpers:run_steps(Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()). + +init_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_started(Config, Testcase). + +end_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_finished(Config, Testcase). + +%% ------------------------------------------------------------------- +%% Test cases. +%% ------------------------------------------------------------------- + +cluster_full_partition_with_autoheal(Config) -> + VHost = <<"/">>, + rabbit_ct_broker_helpers:set_partition_handling_mode_globally(Config, autoheal), + + ?assertEqual(0, count_connections_in(Config, VHost)), + [A, B, C] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + %% 6 connections, 2 per node + Conn1 = open_unmanaged_connection(Config, A), + Conn2 = open_unmanaged_connection(Config, A), + Conn3 = open_unmanaged_connection(Config, B), + Conn4 = open_unmanaged_connection(Config, B), + Conn5 = open_unmanaged_connection(Config, C), + Conn6 = open_unmanaged_connection(Config, C), + wait_for_count_connections_in(Config, VHost, 6, 60000), + + %% B drops off the network, non-reachable by either A or C + rabbit_ct_broker_helpers:block_traffic_between(A, B), + rabbit_ct_broker_helpers:block_traffic_between(B, C), + timer:sleep(?DELAY), + + %% A and C are still connected, so 4 connections are tracked + wait_for_count_connections_in(Config, VHost, 4, 60000), + + rabbit_ct_broker_helpers:allow_traffic_between(A, B), + rabbit_ct_broker_helpers:allow_traffic_between(B, C), + timer:sleep(?DELAY), + + %% during autoheal B's connections were dropped + wait_for_count_connections_in(Config, VHost, 4, 60000), + + lists:foreach(fun (Conn) -> + (catch rabbit_ct_client_helpers:close_connection(Conn)) + end, [Conn1, Conn2, Conn3, Conn4, + Conn5, Conn6]), + + passed. + +%% ------------------------------------------------------------------- +%% Helpers +%% ------------------------------------------------------------------- + +wait_for_count_connections_in(Config, VHost, Expected, Time) when Time =< 0 -> + ?assertMatch(Connections when length(Connections) == Expected, + connections_in(Config, VHost)); +wait_for_count_connections_in(Config, VHost, Expected, Time) -> + case connections_in(Config, VHost) of + Connections when length(Connections) == Expected -> + ok; + _ -> + Sleep = 3000, + timer:sleep(Sleep), + wait_for_count_connections_in(Config, VHost, Expected, Time - Sleep) + end. + +count_connections_in(Config, VHost) -> + count_connections_in(Config, VHost, 0). +count_connections_in(Config, VHost, NodeIndex) -> + rabbit_ct_broker_helpers:rpc(Config, NodeIndex, + rabbit_connection_tracking, + count_tracked_items_in, [{vhost, VHost}]). + +connections_in(Config, VHost) -> + connections_in(Config, 0, VHost). +connections_in(Config, NodeIndex, VHost) -> + rabbit_ct_broker_helpers:rpc(Config, NodeIndex, + rabbit_connection_tracking, + list, [VHost]). diff --git a/deps/rabbit/test/per_vhost_msg_store_SUITE.erl b/deps/rabbit/test/per_vhost_msg_store_SUITE.erl new file mode 100644 index 0000000000..8364d69462 --- /dev/null +++ b/deps/rabbit/test/per_vhost_msg_store_SUITE.erl @@ -0,0 +1,245 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(per_vhost_msg_store_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). + +-compile(export_all). + +-define(MSGS_COUNT, 100). + +all() -> + [ + publish_to_different_dirs, + storage_deleted_on_vhost_delete, + single_vhost_storage_delete_is_safe + ]. + + + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + Config1 = rabbit_ct_helpers:set_config( + Config, + [{rmq_nodename_suffix, ?MODULE}]), + Config2 = rabbit_ct_helpers:merge_app_env( + Config1, + {rabbit, [{queue_index_embed_msgs_below, 100}]}), + rabbit_ct_helpers:run_setup_steps( + Config2, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps( + Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()). + +init_per_testcase(_, Config) -> + Vhost1 = <<"vhost1">>, + Vhost2 = <<"vhost2">>, + rabbit_ct_broker_helpers:add_vhost(Config, Vhost1), + rabbit_ct_broker_helpers:add_vhost(Config, Vhost2), + Chan1 = open_channel(Vhost1, Config), + Chan2 = open_channel(Vhost2, Config), + rabbit_ct_helpers:set_config( + Config, + [{vhost1, Vhost1}, {vhost2, Vhost2}, + {channel1, Chan1}, {channel2, Chan2}]). + +end_per_testcase(single_vhost_storage_delete_is_safe, Config) -> + Config; +end_per_testcase(_, Config) -> + Vhost1 = ?config(vhost1, Config), + Vhost2 = ?config(vhost2, Config), + rabbit_ct_broker_helpers:delete_vhost(Config, Vhost1), + rabbit_ct_broker_helpers:delete_vhost(Config, Vhost2), + Config. + +publish_to_different_dirs(Config) -> + Vhost1 = ?config(vhost1, Config), + Vhost2 = ?config(vhost2, Config), + Channel1 = ?config(channel1, Config), + Channel2 = ?config(channel2, Config), + Queue1 = declare_durable_queue(Channel1), + Queue2 = declare_durable_queue(Channel2), + FolderSize1 = get_folder_size(Vhost1, Config), + FolderSize2 = get_folder_size(Vhost2, Config), + + %% Publish message to a queue index + publish_persistent_messages(index, Channel1, Queue1), + %% First storage increased + FolderSize11 = get_folder_size(Vhost1, Config), + true = (FolderSize1 < FolderSize11), + %% Second storage didn't increased + FolderSize2 = get_folder_size(Vhost2, Config), + + %% Publish message to a message store + publish_persistent_messages(store, Channel1, Queue1), + %% First storage increased + FolderSize12 = get_folder_size(Vhost1, Config), + true = (FolderSize11 < FolderSize12), + %% Second storage didn't increased + FolderSize2 = get_folder_size(Vhost2, Config), + + %% Publish message to a queue index + publish_persistent_messages(index, Channel2, Queue2), + %% First storage increased + FolderSize21 = get_folder_size(Vhost2, Config), + true = (FolderSize2 < FolderSize21), + %% Second storage didn't increased + FolderSize12 = get_folder_size(Vhost1, Config), + + %% Publish message to a message store + publish_persistent_messages(store, Channel2, Queue2), + %% Second storage increased + FolderSize22 = get_folder_size(Vhost2, Config), + true = (FolderSize21 < FolderSize22), + %% First storage didn't increased + FolderSize12 = get_folder_size(Vhost1, Config). + +storage_deleted_on_vhost_delete(Config) -> + Vhost1 = ?config(vhost1, Config), + Channel1 = ?config(channel1, Config), + Queue1 = declare_durable_queue(Channel1), + FolderSize = get_global_folder_size(Config), + + publish_persistent_messages(index, Channel1, Queue1), + publish_persistent_messages(store, Channel1, Queue1), + FolderSizeAfterPublish = get_global_folder_size(Config), + + %% Total storage size increased + true = (FolderSize < FolderSizeAfterPublish), + + ok = rabbit_ct_broker_helpers:delete_vhost(Config, Vhost1), + + %% Total memory reduced + FolderSizeAfterDelete = get_global_folder_size(Config), + true = (FolderSizeAfterPublish > FolderSizeAfterDelete), + + %% There is no Vhost1 folder + 0 = get_folder_size(Vhost1, Config). + + +single_vhost_storage_delete_is_safe(Config) -> +ct:pal("Start test 3", []), + Vhost1 = ?config(vhost1, Config), + Vhost2 = ?config(vhost2, Config), + Channel1 = ?config(channel1, Config), + Channel2 = ?config(channel2, Config), + Queue1 = declare_durable_queue(Channel1), + Queue2 = declare_durable_queue(Channel2), + + %% Publish messages to both stores + publish_persistent_messages(index, Channel1, Queue1), + publish_persistent_messages(store, Channel1, Queue1), + publish_persistent_messages(index, Channel2, Queue2), + publish_persistent_messages(store, Channel2, Queue2), + + queue_is_not_empty(Channel2, Queue2), + % Vhost2Dir = vhost_dir(Vhost2, Config), + % [StoreFile] = filelib:wildcard(binary_to_list(filename:join([Vhost2Dir, "msg_store_persistent_*", "0.rdq"]))), + % ct:pal("Store file ~p~n", [file:read_file(StoreFile)]). +% ok. + rabbit_ct_broker_helpers:stop_broker(Config, 0), + delete_vhost_data(Vhost1, Config), + rabbit_ct_broker_helpers:start_broker(Config, 0), + + Channel11 = open_channel(Vhost1, Config), + Channel21 = open_channel(Vhost2, Config), + + %% There are no Vhost1 messages + queue_is_empty(Channel11, Queue1), + + %% But Vhost2 messages are in place + queue_is_not_empty(Channel21, Queue2), + consume_messages(index, Channel21, Queue2), + consume_messages(store, Channel21, Queue2). + +declare_durable_queue(Channel) -> + QName = list_to_binary(erlang:ref_to_list(make_ref())), + #'queue.declare_ok'{queue = QName} = + amqp_channel:call(Channel, + #'queue.declare'{queue = QName, durable = true}), + QName. + +publish_persistent_messages(Storage, Channel, Queue) -> + MessagePayload = case Storage of + index -> binary:copy(<<"=">>, 50); + store -> binary:copy(<<"-">>, 150) + end, + amqp_channel:call(Channel, #'confirm.select'{}), + [amqp_channel:call(Channel, + #'basic.publish'{routing_key = Queue}, + #amqp_msg{props = #'P_basic'{delivery_mode = 2}, + payload = MessagePayload}) + || _ <- lists:seq(1, ?MSGS_COUNT)], + amqp_channel:wait_for_confirms(Channel). + + +get_folder_size(Vhost, Config) -> + Dir = vhost_dir(Vhost, Config), + folder_size(Dir). + +folder_size(Dir) -> + filelib:fold_files(Dir, ".*", true, + fun(F,Acc) -> filelib:file_size(F) + Acc end, 0). + +get_global_folder_size(Config) -> + BaseDir = rabbit_ct_broker_helpers:rpc(Config, 0, rabbit_mnesia, dir, []), + folder_size(BaseDir). + +vhost_dir(Vhost, Config) -> + rabbit_ct_broker_helpers:rpc(Config, 0, + rabbit_vhost, msg_store_dir_path, [Vhost]). + +delete_vhost_data(Vhost, Config) -> + Dir = vhost_dir(Vhost, Config), + rabbit_file:recursive_delete([Dir]). + +queue_is_empty(Channel, Queue) -> + #'queue.declare_ok'{queue = Queue, message_count = 0} = + amqp_channel:call(Channel, + #'queue.declare'{ queue = Queue, + durable = true, + passive = true}). + +queue_is_not_empty(Channel, Queue) -> + #'queue.declare_ok'{queue = Queue, message_count = MsgCount} = + amqp_channel:call(Channel, + #'queue.declare'{ queue = Queue, + durable = true, + passive = true}), + ExpectedCount = ?MSGS_COUNT * 2, + ExpectedCount = MsgCount. + +consume_messages(Storage, Channel, Queue) -> + MessagePayload = case Storage of + index -> binary:copy(<<"=">>, 50); + store -> binary:copy(<<"-">>, 150) + end, + lists:foreach( + fun(I) -> + ct:pal("Consume message ~p~n ~p~n", [I, MessagePayload]), + {#'basic.get_ok'{}, Content} = + amqp_channel:call(Channel, + #'basic.get'{queue = Queue, + no_ack = true}), + #amqp_msg{payload = MessagePayload} = Content + end, + lists:seq(1, ?MSGS_COUNT)), + ok. + +open_channel(Vhost, Config) -> + Node = rabbit_ct_broker_helpers:get_node_config(Config, 0, nodename), + {ok, Conn} = amqp_connection:start( + #amqp_params_direct{node = Node, virtual_host = Vhost}), + {ok, Chan} = amqp_connection:open_channel(Conn), + Chan. diff --git a/deps/rabbit/test/per_vhost_queue_limit_SUITE.erl b/deps/rabbit/test/per_vhost_queue_limit_SUITE.erl new file mode 100644 index 0000000000..28a9f98537 --- /dev/null +++ b/deps/rabbit/test/per_vhost_queue_limit_SUITE.erl @@ -0,0 +1,682 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2011-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(per_vhost_queue_limit_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-compile(export_all). + +-import(rabbit_ct_client_helpers, [open_unmanaged_connection/3, + close_connection_and_channel/2]). + +all() -> + [ + {group, cluster_size_1} + , {group, cluster_size_2} + ]. + +groups() -> + [ + {cluster_size_1, [], [ + most_basic_single_node_queue_count, + single_node_single_vhost_queue_count, + single_node_multiple_vhosts_queue_count, + single_node_single_vhost_limit, + single_node_single_vhost_zero_limit, + single_node_single_vhost_limit_with_durable_named_queue, + single_node_single_vhost_zero_limit_with_durable_named_queue, + single_node_single_vhost_limit_with_queue_ttl, + single_node_single_vhost_limit_with_redeclaration + ]}, + {cluster_size_2, [], [ + most_basic_cluster_queue_count, + cluster_multiple_vhosts_queue_count, + cluster_multiple_vhosts_limit, + cluster_multiple_vhosts_zero_limit, + cluster_multiple_vhosts_limit_with_durable_named_queue, + cluster_multiple_vhosts_zero_limit_with_durable_named_queue, + cluster_node_restart_queue_count + ]} + ]. + +suite() -> + [ + %% If a test hangs, no need to wait for 30 minutes. + {timetrap, {minutes, 8}} + ]. + +%% ------------------------------------------------------------------- +%% Testsuite setup/teardown. +%% ------------------------------------------------------------------- + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(cluster_size_1, Config) -> + init_per_multinode_group(cluster_size_1, Config, 1); +init_per_group(cluster_size_2, Config) -> + init_per_multinode_group(cluster_size_2, Config, 2); +init_per_group(cluster_rename, Config) -> + init_per_multinode_group(cluster_rename, Config, 2). + +init_per_multinode_group(Group, Config, NodeCount) -> + Suffix = rabbit_ct_helpers:testcase_absname(Config, "", "-"), + Config1 = rabbit_ct_helpers:set_config(Config, [ + {rmq_nodes_count, NodeCount}, + {rmq_nodename_suffix, Suffix} + ]), + case Group of + cluster_rename -> + % The broker is managed by {init,end}_per_testcase(). + Config1; + _ -> + rabbit_ct_helpers:run_steps(Config1, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()) + end. + +end_per_group(cluster_rename, Config) -> + % The broker is managed by {init,end}_per_testcase(). + Config; +end_per_group(_Group, Config) -> + rabbit_ct_helpers:run_steps(Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()). + +init_per_testcase(vhost_limit_after_node_renamed = Testcase, Config) -> + rabbit_ct_helpers:testcase_started(Config, Testcase), + rabbit_ct_helpers:run_steps(Config, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()); +init_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_started(Config, Testcase), + Config. + +end_per_testcase(vhost_limit_after_node_renamed = Testcase, Config) -> + Config1 = ?config(save_config, Config), + rabbit_ct_helpers:run_steps(Config1, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()), + rabbit_ct_helpers:testcase_finished(Config1, Testcase); +end_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_finished(Config, Testcase). + +%% ------------------------------------------------------------------- +%% Test cases. +%% ------------------------------------------------------------------- + +most_basic_single_node_queue_count(Config) -> + VHost = <<"queue-limits">>, + set_up_vhost(Config, VHost), + ?assertEqual(0, count_queues_in(Config, VHost)), + Conn = open_unmanaged_connection(Config, 0, VHost), + {ok, Ch} = amqp_connection:open_channel(Conn), + declare_exclusive_queues(Ch, 10), + ?assertEqual(10, count_queues_in(Config, VHost)), + close_connection_and_channel(Conn, Ch), + ?assertEqual(0, count_queues_in(Config, VHost)), + rabbit_ct_broker_helpers:delete_vhost(Config, VHost). + +single_node_single_vhost_queue_count(Config) -> + VHost = <<"queue-limits">>, + set_up_vhost(Config, VHost), + ?assertEqual(0, count_queues_in(Config, VHost)), + Conn = open_unmanaged_connection(Config, 0, VHost), + {ok, Ch} = amqp_connection:open_channel(Conn), + declare_exclusive_queues(Ch, 10), + ?assertEqual(10, count_queues_in(Config, VHost)), + declare_durable_queues(Ch, 10), + ?assertEqual(20, count_queues_in(Config, VHost)), + delete_durable_queues(Ch, 10), + ?assertEqual(10, count_queues_in(Config, VHost)), + close_connection_and_channel(Conn, Ch), + ?assertEqual(0, count_queues_in(Config, VHost)), + rabbit_ct_broker_helpers:delete_vhost(Config, VHost). + +single_node_multiple_vhosts_queue_count(Config) -> + VHost1 = <<"queue-limits1">>, + VHost2 = <<"queue-limits2">>, + set_up_vhost(Config, VHost1), + set_up_vhost(Config, VHost2), + + ?assertEqual(0, count_queues_in(Config, VHost1)), + ?assertEqual(0, count_queues_in(Config, VHost2)), + + Conn1 = open_unmanaged_connection(Config, 0, VHost1), + {ok, Ch1} = amqp_connection:open_channel(Conn1), + Conn2 = open_unmanaged_connection(Config, 0, VHost2), + {ok, Ch2} = amqp_connection:open_channel(Conn2), + + declare_exclusive_queues(Ch1, 10), + ?assertEqual(10, count_queues_in(Config, VHost1)), + declare_durable_queues(Ch1, 10), + ?assertEqual(20, count_queues_in(Config, VHost1)), + delete_durable_queues(Ch1, 10), + ?assertEqual(10, count_queues_in(Config, VHost1)), + declare_exclusive_queues(Ch2, 30), + ?assertEqual(30, count_queues_in(Config, VHost2)), + close_connection_and_channel(Conn1, Ch1), + ?assertEqual(0, count_queues_in(Config, VHost1)), + close_connection_and_channel(Conn2, Ch2), + ?assertEqual(0, count_queues_in(Config, VHost2)), + rabbit_ct_broker_helpers:delete_vhost(Config, VHost1), + rabbit_ct_broker_helpers:delete_vhost(Config, VHost2). + +single_node_single_vhost_limit(Config) -> + single_node_single_vhost_limit_with(Config, 5), + single_node_single_vhost_limit_with(Config, 10). +single_node_single_vhost_zero_limit(Config) -> + single_node_single_vhost_zero_limit_with(Config, #'queue.declare'{queue = <<"">>, + exclusive = true}). + +single_node_single_vhost_limit_with_durable_named_queue(Config) -> + VHost = <<"queue-limits">>, + set_up_vhost(Config, VHost), + ?assertEqual(0, count_queues_in(Config, VHost)), + + set_vhost_queue_limit(Config, VHost, 3), + Conn = open_unmanaged_connection(Config, 0, VHost), + {ok, Ch} = amqp_connection:open_channel(Conn), + + #'queue.declare_ok'{queue = _} = + amqp_channel:call(Ch, #'queue.declare'{queue = <<"Q1">>, + exclusive = false, + durable = true}), + #'queue.declare_ok'{queue = _} = + amqp_channel:call(Ch, #'queue.declare'{queue = <<"Q2">>, + exclusive = false, + durable = true}), + #'queue.declare_ok'{queue = _} = + amqp_channel:call(Ch, #'queue.declare'{queue = <<"Q3">>, + exclusive = false, + durable = true}), + + expect_shutdown_due_to_precondition_failed( + fun () -> + amqp_channel:call(Ch, #'queue.declare'{queue = <<"Q4">>, + exclusive = false, + durable = true}) + end), + + rabbit_ct_broker_helpers:delete_vhost(Config, VHost). + +single_node_single_vhost_zero_limit_with_durable_named_queue(Config) -> + single_node_single_vhost_zero_limit_with(Config, #'queue.declare'{queue = <<"Q4">>, + exclusive = false, + durable = true}). + +single_node_single_vhost_limit_with(Config, WatermarkLimit) -> + VHost = <<"queue-limits">>, + set_up_vhost(Config, VHost), + ?assertEqual(0, count_queues_in(Config, VHost)), + + set_vhost_queue_limit(Config, VHost, 3), + Conn = open_unmanaged_connection(Config, 0, VHost), + {ok, Ch} = amqp_connection:open_channel(Conn), + + set_vhost_queue_limit(Config, VHost, WatermarkLimit), + lists:foreach(fun (_) -> + #'queue.declare_ok'{queue = _} = + amqp_channel:call(Ch, #'queue.declare'{queue = <<"">>, + exclusive = true}) + end, lists:seq(1, WatermarkLimit)), + + expect_shutdown_due_to_precondition_failed( + fun () -> + amqp_channel:call(Ch, #'queue.declare'{queue = <<"">>, + exclusive = true}) + end), + + rabbit_ct_broker_helpers:delete_vhost(Config, VHost). + +single_node_single_vhost_zero_limit_with(Config, QueueDeclare) -> + VHost = <<"queue-limits">>, + set_up_vhost(Config, VHost), + ?assertEqual(0, count_queues_in(Config, VHost)), + + Conn1 = open_unmanaged_connection(Config, 0, VHost), + {ok, Ch1} = amqp_connection:open_channel(Conn1), + + set_vhost_queue_limit(Config, VHost, 0), + + expect_shutdown_due_to_precondition_failed( + fun () -> + amqp_channel:call(Ch1, QueueDeclare) + end), + + Conn2 = open_unmanaged_connection(Config, 0, VHost), + {ok, Ch2} = amqp_connection:open_channel(Conn2), + + %% lift the limit + set_vhost_queue_limit(Config, VHost, -1), + lists:foreach(fun (_) -> + #'queue.declare_ok'{queue = _} = + amqp_channel:call(Ch2, #'queue.declare'{queue = <<"">>, + exclusive = true}) + end, lists:seq(1, 100)), + + rabbit_ct_broker_helpers:delete_vhost(Config, VHost). + + +single_node_single_vhost_limit_with_queue_ttl(Config) -> + VHost = <<"queue-limits">>, + set_up_vhost(Config, VHost), + ?assertEqual(0, count_queues_in(Config, VHost)), + + Conn1 = open_unmanaged_connection(Config, 0, VHost), + {ok, Ch1} = amqp_connection:open_channel(Conn1), + + set_vhost_queue_limit(Config, VHost, 3), + + lists:foreach(fun (_) -> + #'queue.declare_ok'{queue = _} = + amqp_channel:call(Ch1, #'queue.declare'{queue = <<"">>, + exclusive = true, + arguments = [{<<"x-expires">>, long, 2000}]}) + end, lists:seq(1, 3)), + + + expect_shutdown_due_to_precondition_failed( + fun () -> + amqp_channel:call(Ch1, #'queue.declare'{queue = <<"">>, + exclusive = true}) + end), + + Conn2 = open_unmanaged_connection(Config, 0, VHost), + {ok, Ch2} = amqp_connection:open_channel(Conn2), + + %% wait for the queues to expire + timer:sleep(3000), + + #'queue.declare_ok'{queue = _} = + amqp_channel:call(Ch2, #'queue.declare'{queue = <<"">>, + exclusive = true}), + + rabbit_ct_broker_helpers:delete_vhost(Config, VHost). + + +single_node_single_vhost_limit_with_redeclaration(Config) -> + VHost = <<"queue-limits">>, + set_up_vhost(Config, VHost), + ?assertEqual(0, count_queues_in(Config, VHost)), + + set_vhost_queue_limit(Config, VHost, 3), + Conn1 = open_unmanaged_connection(Config, 0, VHost), + {ok, Ch1} = amqp_connection:open_channel(Conn1), + + #'queue.declare_ok'{queue = _} = + amqp_channel:call(Ch1, #'queue.declare'{queue = <<"Q1">>, + exclusive = false, + durable = true}), + #'queue.declare_ok'{queue = _} = + amqp_channel:call(Ch1, #'queue.declare'{queue = <<"Q2">>, + exclusive = false, + durable = true}), + #'queue.declare_ok'{queue = _} = + amqp_channel:call(Ch1, #'queue.declare'{queue = <<"Q3">>, + exclusive = false, + durable = true}), + + %% can't declare a new queue... + expect_shutdown_due_to_precondition_failed( + fun () -> + amqp_channel:call(Ch1, #'queue.declare'{queue = <<"Q4">>, + exclusive = false, + durable = true}) + end), + + + Conn2 = open_unmanaged_connection(Config, 0, VHost), + {ok, Ch2} = amqp_connection:open_channel(Conn2), + + %% ...but re-declarations succeed + #'queue.declare_ok'{queue = _} = + amqp_channel:call(Ch2, #'queue.declare'{queue = <<"Q1">>, + exclusive = false, + durable = true}), + #'queue.declare_ok'{queue = _} = + amqp_channel:call(Ch2, #'queue.declare'{queue = <<"Q2">>, + exclusive = false, + durable = true}), + #'queue.declare_ok'{queue = _} = + amqp_channel:call(Ch2, #'queue.declare'{queue = <<"Q3">>, + exclusive = false, + durable = true}), + + expect_shutdown_due_to_precondition_failed( + fun () -> + amqp_channel:call(Ch2, #'queue.declare'{queue = <<"Q4">>, + exclusive = false, + durable = true}) + end), + + rabbit_ct_broker_helpers:delete_vhost(Config, VHost). + + +most_basic_cluster_queue_count(Config) -> + VHost = <<"queue-limits">>, + set_up_vhost(Config, VHost), + ?assertEqual(0, count_queues_in(Config, VHost, 0)), + ?assertEqual(0, count_queues_in(Config, VHost, 1)), + + Conn1 = open_unmanaged_connection(Config, 0, VHost), + {ok, Ch1} = amqp_connection:open_channel(Conn1), + declare_exclusive_queues(Ch1, 10), + ?assertEqual(10, count_queues_in(Config, VHost, 0)), + ?assertEqual(10, count_queues_in(Config, VHost, 1)), + + Conn2 = open_unmanaged_connection(Config, 0, VHost), + {ok, Ch2} = amqp_connection:open_channel(Conn2), + declare_exclusive_queues(Ch2, 15), + ?assertEqual(25, count_queues_in(Config, VHost, 0)), + ?assertEqual(25, count_queues_in(Config, VHost, 1)), + close_connection_and_channel(Conn1, Ch1), + close_connection_and_channel(Conn2, Ch2), + ?assertEqual(0, count_queues_in(Config, VHost, 0)), + ?assertEqual(0, count_queues_in(Config, VHost, 1)), + rabbit_ct_broker_helpers:delete_vhost(Config, VHost). + +cluster_node_restart_queue_count(Config) -> + VHost = <<"queue-limits">>, + set_up_vhost(Config, VHost), + ?assertEqual(0, count_queues_in(Config, VHost, 0)), + ?assertEqual(0, count_queues_in(Config, VHost, 1)), + + Conn1 = open_unmanaged_connection(Config, 0, VHost), + {ok, Ch1} = amqp_connection:open_channel(Conn1), + declare_exclusive_queues(Ch1, 10), + ?assertEqual(10, count_queues_in(Config, VHost, 0)), + ?assertEqual(10, count_queues_in(Config, VHost, 1)), + + rabbit_ct_broker_helpers:restart_broker(Config, 0), + ?assertEqual(0, count_queues_in(Config, VHost, 0)), + + Conn2 = open_unmanaged_connection(Config, 1, VHost), + {ok, Ch2} = amqp_connection:open_channel(Conn2), + declare_exclusive_queues(Ch2, 15), + ?assertEqual(15, count_queues_in(Config, VHost, 0)), + ?assertEqual(15, count_queues_in(Config, VHost, 1)), + + declare_durable_queues(Ch2, 10), + ?assertEqual(25, count_queues_in(Config, VHost, 0)), + ?assertEqual(25, count_queues_in(Config, VHost, 1)), + + rabbit_ct_broker_helpers:restart_broker(Config, 1), + + ?assertEqual(10, count_queues_in(Config, VHost, 0)), + ?assertEqual(10, count_queues_in(Config, VHost, 1)), + rabbit_ct_broker_helpers:delete_vhost(Config, VHost). + + +cluster_multiple_vhosts_queue_count(Config) -> + VHost1 = <<"queue-limits1">>, + VHost2 = <<"queue-limits2">>, + set_up_vhost(Config, VHost1), + set_up_vhost(Config, VHost2), + + ?assertEqual(0, count_queues_in(Config, VHost1)), + ?assertEqual(0, count_queues_in(Config, VHost2)), + + Conn1 = open_unmanaged_connection(Config, 0, VHost1), + {ok, Ch1} = amqp_connection:open_channel(Conn1), + declare_exclusive_queues(Ch1, 10), + ?assertEqual(10, count_queues_in(Config, VHost1, 0)), + ?assertEqual(10, count_queues_in(Config, VHost1, 1)), + ?assertEqual(0, count_queues_in(Config, VHost2, 0)), + ?assertEqual(0, count_queues_in(Config, VHost2, 1)), + + Conn2 = open_unmanaged_connection(Config, 0, VHost2), + {ok, Ch2} = amqp_connection:open_channel(Conn2), + declare_exclusive_queues(Ch2, 15), + ?assertEqual(15, count_queues_in(Config, VHost2, 0)), + ?assertEqual(15, count_queues_in(Config, VHost2, 1)), + close_connection_and_channel(Conn1, Ch1), + close_connection_and_channel(Conn2, Ch2), + ?assertEqual(0, count_queues_in(Config, VHost1, 0)), + ?assertEqual(0, count_queues_in(Config, VHost1, 1)), + ?assertEqual(0, count_queues_in(Config, VHost2, 0)), + ?assertEqual(0, count_queues_in(Config, VHost2, 1)), + + rabbit_ct_broker_helpers:delete_vhost(Config, VHost1), + rabbit_ct_broker_helpers:delete_vhost(Config, VHost2). + +cluster_multiple_vhosts_limit(Config) -> + cluster_multiple_vhosts_limit_with(Config, 10), + cluster_multiple_vhosts_limit_with(Config, 20). + +cluster_multiple_vhosts_limit_with(Config, WatermarkLimit) -> + VHost1 = <<"queue-limits1">>, + VHost2 = <<"queue-limits2">>, + set_up_vhost(Config, VHost1), + set_up_vhost(Config, VHost2), + ?assertEqual(0, count_queues_in(Config, VHost1)), + ?assertEqual(0, count_queues_in(Config, VHost2)), + + set_vhost_queue_limit(Config, VHost1, 3), + set_vhost_queue_limit(Config, VHost2, 3), + + Conn1 = open_unmanaged_connection(Config, 0, VHost1), + {ok, Ch1} = amqp_connection:open_channel(Conn1), + set_vhost_queue_limit(Config, VHost1, WatermarkLimit), + + lists:foreach(fun (_) -> + #'queue.declare_ok'{queue = _} = + amqp_channel:call(Ch1, #'queue.declare'{queue = <<"">>, + exclusive = true}) + end, lists:seq(1, WatermarkLimit)), + + Conn2 = open_unmanaged_connection(Config, 1, VHost2), + {ok, Ch2} = amqp_connection:open_channel(Conn2), + set_vhost_queue_limit(Config, VHost2, WatermarkLimit), + + lists:foreach(fun (_) -> + #'queue.declare_ok'{queue = _} = + amqp_channel:call(Ch2, #'queue.declare'{queue = <<"">>, + exclusive = true}) + end, lists:seq(1, WatermarkLimit)), + + expect_shutdown_due_to_precondition_failed( + fun () -> + amqp_channel:call(Ch1, #'queue.declare'{queue = <<"">>, + exclusive = true}) + end), + expect_shutdown_due_to_precondition_failed( + fun () -> + amqp_channel:call(Ch2, #'queue.declare'{queue = <<"">>, + exclusive = true}) + end), + + rabbit_ct_broker_helpers:delete_vhost(Config, VHost1), + rabbit_ct_broker_helpers:delete_vhost(Config, VHost2). + + +cluster_multiple_vhosts_zero_limit(Config) -> + cluster_multiple_vhosts_zero_limit_with(Config, #'queue.declare'{queue = <<"">>, + exclusive = true}). + +cluster_multiple_vhosts_limit_with_durable_named_queue(Config) -> + VHost1 = <<"queue-limits1">>, + VHost2 = <<"queue-limits2">>, + set_up_vhost(Config, VHost1), + set_up_vhost(Config, VHost2), + ?assertEqual(0, count_queues_in(Config, VHost1)), + ?assertEqual(0, count_queues_in(Config, VHost2)), + + set_vhost_queue_limit(Config, VHost1, 3), + set_vhost_queue_limit(Config, VHost2, 3), + + Conn1 = open_unmanaged_connection(Config, 0, VHost1), + {ok, Ch1} = amqp_connection:open_channel(Conn1), + + Conn2 = open_unmanaged_connection(Config, 1, VHost2), + {ok, Ch2} = amqp_connection:open_channel(Conn2), + + #'queue.declare_ok'{} = + amqp_channel:call(Ch1, #'queue.declare'{queue = <<"Q1">>, + exclusive = false, + durable = true}), + #'queue.declare_ok'{} = + amqp_channel:call(Ch1, #'queue.declare'{queue = <<"Q2">>, + exclusive = false, + durable = true}), + #'queue.declare_ok'{} = + amqp_channel:call(Ch1, #'queue.declare'{queue = <<"Q3">>, + exclusive = false, + durable = true}), + + #'queue.declare_ok'{} = + amqp_channel:call(Ch2, #'queue.declare'{queue = <<"Q1">>, + exclusive = false, + durable = true}), + #'queue.declare_ok'{} = + amqp_channel:call(Ch2, #'queue.declare'{queue = <<"Q2">>, + exclusive = false, + durable = true}), + #'queue.declare_ok'{} = + amqp_channel:call(Ch2, #'queue.declare'{queue = <<"Q3">>, + exclusive = false, + durable = true}), + + expect_shutdown_due_to_precondition_failed( + fun () -> + amqp_channel:call(Ch1, #'queue.declare'{queue = <<"Q3">>, + exclusive = false, + durable = true}) + end), + expect_shutdown_due_to_precondition_failed( + fun () -> + amqp_channel:call(Ch2, #'queue.declare'{queue = <<"Q3">>, + exclusive = false, + durable = true}) + end), + + rabbit_ct_broker_helpers:delete_vhost(Config, VHost1), + rabbit_ct_broker_helpers:delete_vhost(Config, VHost2). + +cluster_multiple_vhosts_zero_limit_with_durable_named_queue(Config) -> + cluster_multiple_vhosts_zero_limit_with(Config, #'queue.declare'{queue = <<"Q4">>, + exclusive = false, + durable = true}). + +cluster_multiple_vhosts_zero_limit_with(Config, QueueDeclare) -> + VHost1 = <<"queue-limits1">>, + VHost2 = <<"queue-limits2">>, + set_up_vhost(Config, VHost1), + set_up_vhost(Config, VHost2), + ?assertEqual(0, count_queues_in(Config, VHost1)), + ?assertEqual(0, count_queues_in(Config, VHost2)), + + Conn1 = open_unmanaged_connection(Config, 0, VHost1), + {ok, Ch1} = amqp_connection:open_channel(Conn1), + Conn2 = open_unmanaged_connection(Config, 1, VHost2), + {ok, Ch2} = amqp_connection:open_channel(Conn2), + + set_vhost_queue_limit(Config, VHost1, 0), + set_vhost_queue_limit(Config, VHost2, 0), + + expect_shutdown_due_to_precondition_failed( + fun () -> + amqp_channel:call(Ch1, QueueDeclare) + end), + expect_shutdown_due_to_precondition_failed( + fun () -> + amqp_channel:call(Ch2, QueueDeclare) + end), + + + Conn3 = open_unmanaged_connection(Config, 0, VHost1), + {ok, Ch3} = amqp_connection:open_channel(Conn3), + Conn4 = open_unmanaged_connection(Config, 1, VHost2), + {ok, Ch4} = amqp_connection:open_channel(Conn4), + + %% lift the limits + set_vhost_queue_limit(Config, VHost1, -1), + set_vhost_queue_limit(Config, VHost2, -1), + lists:foreach(fun (_) -> + #'queue.declare_ok'{queue = _} = + amqp_channel:call(Ch3, #'queue.declare'{queue = <<"">>, + exclusive = true}), + #'queue.declare_ok'{queue = _} = + amqp_channel:call(Ch4, #'queue.declare'{queue = <<"">>, + exclusive = true}) + end, lists:seq(1, 400)), + + rabbit_ct_broker_helpers:delete_vhost(Config, VHost1), + rabbit_ct_broker_helpers:delete_vhost(Config, VHost2). + + + +%% ------------------------------------------------------------------- +%% Helpers +%% ------------------------------------------------------------------- + +set_up_vhost(Config, VHost) -> + rabbit_ct_broker_helpers:add_vhost(Config, VHost), + rabbit_ct_broker_helpers:set_full_permissions(Config, <<"guest">>, VHost), + set_vhost_queue_limit(Config, VHost, -1). + +set_vhost_queue_limit(Config, VHost, Count) -> + set_vhost_queue_limit(Config, 0, VHost, Count). + +set_vhost_queue_limit(Config, NodeIndex, VHost, Count) -> + Node = rabbit_ct_broker_helpers:get_node_config( + Config, NodeIndex, nodename), + rabbit_ct_broker_helpers:control_action( + set_vhost_limits, Node, + ["{\"max-queues\": " ++ integer_to_list(Count) ++ "}"], + [{"-p", binary_to_list(VHost)}]). + +count_queues_in(Config, VHost) -> + count_queues_in(Config, VHost, 0). +count_queues_in(Config, VHost, NodeIndex) -> + timer:sleep(200), + rabbit_ct_broker_helpers:rpc(Config, NodeIndex, + rabbit_amqqueue, + count, [VHost]). + +declare_exclusive_queues(Ch, N) -> + lists:foreach(fun (_) -> + amqp_channel:call(Ch, + #'queue.declare'{queue = <<"">>, + exclusive = true}) + end, + lists:seq(1, N)). + +declare_durable_queues(Ch, N) -> + lists:foreach(fun (I) -> + amqp_channel:call(Ch, + #'queue.declare'{queue = durable_queue_name(I), + exclusive = false, + durable = true}) + end, + lists:seq(1, N)). + +delete_durable_queues(Ch, N) -> + lists:foreach(fun (I) -> + amqp_channel:call(Ch, + #'queue.delete'{queue = durable_queue_name(I)}) + end, + lists:seq(1, N)). + +durable_queue_name(N) when is_integer(N) -> + iolist_to_binary(io_lib:format("queue-limits-durable-~p", [N])). + +expect_shutdown_due_to_precondition_failed(Thunk) -> + try + Thunk(), + ok + catch _:{{shutdown, {server_initiated_close, 406, _}}, _} -> + %% expected + ok + end. diff --git a/deps/rabbit/test/policy_SUITE.erl b/deps/rabbit/test/policy_SUITE.erl new file mode 100644 index 0000000000..ce68332d77 --- /dev/null +++ b/deps/rabbit/test/policy_SUITE.erl @@ -0,0 +1,204 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2011-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(policy_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). + +-compile(export_all). + +all() -> + [ + {group, cluster_size_2} + ]. + +groups() -> + [ + {cluster_size_2, [], [ + policy_ttl, + operator_policy_ttl, + operator_retroactive_policy_ttl, + operator_retroactive_policy_publish_ttl + ]} + ]. + +%% ------------------------------------------------------------------- +%% Testsuite setup/teardown. +%% ------------------------------------------------------------------- + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(cluster_size_2, Config) -> + Suffix = rabbit_ct_helpers:testcase_absname(Config, "", "-"), + Config1 = rabbit_ct_helpers:set_config(Config, [ + {rmq_nodes_count, 2}, + {rmq_nodename_suffix, Suffix} + ]), + rabbit_ct_helpers:run_steps(Config1, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()). + +end_per_group(_Group, Config) -> + rabbit_ct_helpers:run_steps(Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()). + +init_per_testcase(Testcase, Config) -> + rabbit_ct_client_helpers:setup_steps(), + rabbit_ct_helpers:testcase_started(Config, Testcase). + +end_per_testcase(Testcase, Config) -> + rabbit_ct_client_helpers:teardown_steps(), + rabbit_ct_helpers:testcase_finished(Config, Testcase). + +%% ------------------------------------------------------------------- +%% Test cases. +%% ------------------------------------------------------------------- + +policy_ttl(Config) -> + {Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + Q = <<"policy_ttl-queue">>, + rabbit_ct_broker_helpers:set_policy(Config, 0, <<"ttl-policy">>, + <<"policy_ttl-queue">>, <<"all">>, [{<<"message-ttl">>, 20}]), + + declare(Ch, Q), + publish(Ch, Q, lists:seq(1, 20)), + timer:sleep(50), + get_empty(Ch, Q), + delete(Ch, Q), + + rabbit_ct_broker_helpers:clear_policy(Config, 0, <<"ttl-policy">>), + + rabbit_ct_client_helpers:close_channel(Ch), + rabbit_ct_client_helpers:close_connection(Conn), + passed. + +operator_policy_ttl(Config) -> + {Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + Q = <<"policy_ttl-queue">>, + % Operator policy will override + rabbit_ct_broker_helpers:set_policy(Config, 0, <<"ttl-policy">>, + <<"policy_ttl-queue">>, <<"all">>, [{<<"message-ttl">>, 100000}]), + rabbit_ct_broker_helpers:set_operator_policy(Config, 0, <<"ttl-policy-op">>, + <<"policy_ttl-queue">>, <<"all">>, [{<<"message-ttl">>, 1}]), + + declare(Ch, Q), + publish(Ch, Q, lists:seq(1, 50)), + timer:sleep(50), + get_empty(Ch, Q), + delete(Ch, Q), + + rabbit_ct_broker_helpers:clear_policy(Config, 0, <<"ttl-policy">>), + rabbit_ct_broker_helpers:clear_operator_policy(Config, 0, <<"ttl-policy-op">>), + + rabbit_ct_client_helpers:close_channel(Ch), + rabbit_ct_client_helpers:close_connection(Conn), + passed. + +operator_retroactive_policy_ttl(Config) -> + {Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + Q = <<"policy_ttl-queue">>, + declare(Ch, Q), + publish(Ch, Q, lists:seq(1, 50)), + % Operator policy will override + rabbit_ct_broker_helpers:set_operator_policy(Config, 0, <<"ttl-policy-op">>, + <<"policy_ttl-queue">>, <<"all">>, [{<<"message-ttl">>, 1}]), + + %% Old messages are not expired + timer:sleep(50), + get_messages(50, Ch, Q), + delete(Ch, Q), + + rabbit_ct_broker_helpers:clear_operator_policy(Config, 0, <<"ttl-policy-op">>), + + rabbit_ct_client_helpers:close_channel(Ch), + rabbit_ct_client_helpers:close_connection(Conn), + passed. + +operator_retroactive_policy_publish_ttl(Config) -> + {Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + Q = <<"policy_ttl-queue">>, + declare(Ch, Q), + publish(Ch, Q, lists:seq(1, 50)), + % Operator policy will override + rabbit_ct_broker_helpers:set_operator_policy(Config, 0, <<"ttl-policy-op">>, + <<"policy_ttl-queue">>, <<"all">>, [{<<"message-ttl">>, 1}]), + + %% Old messages are not expired, new ones only expire when they get to the head of + %% the queue + publish(Ch, Q, lists:seq(1, 25)), + timer:sleep(50), + [[<<"policy_ttl-queue">>, <<"75">>]] = + rabbit_ct_broker_helpers:rabbitmqctl_list(Config, 0, ["list_queues", "--no-table-headers"]), + get_messages(50, Ch, Q), + delete(Ch, Q), + + rabbit_ct_broker_helpers:clear_operator_policy(Config, 0, <<"ttl-policy-op">>), + + rabbit_ct_client_helpers:close_channel(Ch), + rabbit_ct_client_helpers:close_connection(Conn), + passed. + +%%---------------------------------------------------------------------------- + + +declare(Ch, Q) -> + amqp_channel:call(Ch, #'queue.declare'{queue = Q, + durable = true}). + +delete(Ch, Q) -> + amqp_channel:call(Ch, #'queue.delete'{queue = Q}). + +publish(Ch, Q, Ps) -> + amqp_channel:call(Ch, #'confirm.select'{}), + [publish1(Ch, Q, P) || P <- Ps], + amqp_channel:wait_for_confirms(Ch). + +publish1(Ch, Q, P) -> + amqp_channel:cast(Ch, #'basic.publish'{routing_key = Q}, + #amqp_msg{props = props(P), + payload = erlang:md5(term_to_binary(P))}). + +publish1(Ch, Q, P, Pd) -> + amqp_channel:cast(Ch, #'basic.publish'{routing_key = Q}, + #amqp_msg{props = props(P), + payload = Pd}). + +props(undefined) -> #'P_basic'{delivery_mode = 2}; +props(P) -> #'P_basic'{priority = P, + delivery_mode = 2}. + +consume(Ch, Q, Ack) -> + amqp_channel:subscribe(Ch, #'basic.consume'{queue = Q, + no_ack = Ack =:= no_ack, + consumer_tag = <<"ctag">>}, + self()), + receive + #'basic.consume_ok'{consumer_tag = <<"ctag">>} -> + ok + end. + +get_empty(Ch, Q) -> + #'basic.get_empty'{} = amqp_channel:call(Ch, #'basic.get'{queue = Q}). + +get_messages(0, Ch, Q) -> + get_empty(Ch, Q); +get_messages(Number, Ch, Q) -> + case amqp_channel:call(Ch, #'basic.get'{queue = Q}) of + {#'basic.get_ok'{}, _} -> + get_messages(Number - 1, Ch, Q); + #'basic.get_empty'{} -> + exit(failed) + end. + +%%---------------------------------------------------------------------------- diff --git a/deps/rabbit/test/priority_queue_SUITE.erl b/deps/rabbit/test/priority_queue_SUITE.erl new file mode 100644 index 0000000000..a0c1732ffd --- /dev/null +++ b/deps/rabbit/test/priority_queue_SUITE.erl @@ -0,0 +1,771 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(priority_queue_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). + +-compile(export_all). + +all() -> + [ + {group, cluster_size_2}, + {group, cluster_size_3} + ]. + +groups() -> + [ + {cluster_size_2, [], [ + ackfold, + drop, + {overflow_reject_publish, [], [reject]}, + {overflow_reject_publish_dlx, [], [reject]}, + dropwhile_fetchwhile, + info_head_message_timestamp, + matching, + mirror_queue_sync, + mirror_queue_sync_priority_above_max, + mirror_queue_sync_priority_above_max_pending_ack, + mirror_queue_sync_order, + purge, + requeue, + resume, + simple_order, + straight_through, + invoke, + gen_server2_stats, + negative_max_priorities, + max_priorities_above_hard_limit + ]}, + {cluster_size_3, [], [ + mirror_queue_auto_ack, + mirror_fast_reset_policy, + mirror_reset_policy, + mirror_stop_pending_followers + ]} + ]. + +%% ------------------------------------------------------------------- +%% Testsuite setup/teardown. +%% ------------------------------------------------------------------- + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(cluster_size_2, Config) -> + Suffix = rabbit_ct_helpers:testcase_absname(Config, "", "-"), + Config1 = rabbit_ct_helpers:set_config(Config, [ + {rmq_nodes_count, 2}, + {rmq_nodename_suffix, Suffix} + ]), + rabbit_ct_helpers:run_steps(Config1, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()); +init_per_group(cluster_size_3, Config) -> + Suffix = rabbit_ct_helpers:testcase_absname(Config, "", "-"), + Config1 = rabbit_ct_helpers:set_config(Config, [ + {rmq_nodes_count, 3}, + {rmq_nodename_suffix, Suffix} + ]), + rabbit_ct_helpers:run_steps(Config1, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()); +init_per_group(overflow_reject_publish, Config) -> + rabbit_ct_helpers:set_config(Config, [ + {overflow, <<"reject-publish">>} + ]); +init_per_group(overflow_reject_publish_dlx, Config) -> + rabbit_ct_helpers:set_config(Config, [ + {overflow, <<"reject-publish-dlx">>} + ]). + +end_per_group(overflow_reject_publish, _Config) -> + ok; +end_per_group(overflow_reject_publish_dlx, _Config) -> + ok; +end_per_group(_Group, Config) -> + rabbit_ct_helpers:run_steps(Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()). + +init_per_testcase(Testcase, Config) -> + rabbit_ct_client_helpers:setup_steps(), + rabbit_ct_helpers:testcase_started(Config, Testcase). + +end_per_testcase(Testcase, Config) -> + rabbit_ct_client_helpers:teardown_steps(), + rabbit_ct_helpers:testcase_finished(Config, Testcase). + +%% ------------------------------------------------------------------- +%% Testcases. +%% ------------------------------------------------------------------- + +%% The BQ API is used in all sorts of places in all sorts of +%% ways. Therefore we have to jump through a few different hoops +%% in order to integration-test it. +%% +%% * start/1, stop/0, init/3, terminate/2, delete_and_terminate/2 +%% - starting and stopping rabbit. durable queues / persistent msgs needed +%% to test recovery +%% +%% * publish/5, drain_confirmed/1, fetch/2, ack/2, is_duplicate/2, msg_rates/1, +%% needs_timeout/1, timeout/1, invoke/3, resume/1 [0] +%% - regular publishing and consuming, with confirms and acks and durability +%% +%% * publish_delivered/4 - publish with acks straight through +%% * discard/3 - publish without acks straight through +%% * dropwhile/2 - expire messages without DLX +%% * fetchwhile/4 - expire messages with DLX +%% * ackfold/4 - reject messages with DLX +%% * requeue/2 - reject messages without DLX +%% * drop/2 - maxlen messages without DLX +%% * purge/1 - issue AMQP queue.purge +%% * purge_acks/1 - mirror queue explicit sync with unacked msgs +%% * fold/3 - mirror queue explicit sync +%% * depth/1 - mirror queue implicit sync detection +%% * len/1, is_empty/1 - info items +%% * handle_pre_hibernate/1 - hibernation +%% +%% * set_ram_duration_target/2, ram_duration/1, status/1 +%% - maybe need unit testing? +%% +%% [0] publish enough to get credit flow from msg store + +simple_order(Config) -> + {Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + Q = <<"simple_order-queue">>, + declare(Ch, Q, 3), + publish(Ch, Q, [1, 2, 3, 1, 2, 3, 1, 2, 3]), + get_all(Ch, Q, do_ack, [3, 3, 3, 2, 2, 2, 1, 1, 1]), + publish(Ch, Q, [2, 3, 1, 2, 3, 1, 2, 3, 1]), + get_all(Ch, Q, no_ack, [3, 3, 3, 2, 2, 2, 1, 1, 1]), + publish(Ch, Q, [3, 1, 2, 3, 1, 2, 3, 1, 2]), + get_all(Ch, Q, do_ack, [3, 3, 3, 2, 2, 2, 1, 1, 1]), + delete(Ch, Q), + rabbit_ct_client_helpers:close_channel(Ch), + rabbit_ct_client_helpers:close_connection(Conn), + passed. + +matching(Config) -> + {Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + Q = <<"matching-queue">>, + declare(Ch, Q, 5), + %% We round priority down, and 0 is the default + publish(Ch, Q, [undefined, 0, 5, 10, undefined]), + get_all(Ch, Q, do_ack, [5, 10, undefined, 0, undefined]), + delete(Ch, Q), + rabbit_ct_client_helpers:close_channel(Ch), + rabbit_ct_client_helpers:close_connection(Conn), + passed. + +resume(Config) -> + {Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + Q = <<"resume-queue">>, + declare(Ch, Q, 5), + amqp_channel:call(Ch, #'confirm.select'{}), + publish_many(Ch, Q, 10000), + amqp_channel:wait_for_confirms(Ch), + amqp_channel:call(Ch, #'queue.purge'{queue = Q}), %% Assert it exists + delete(Ch, Q), + rabbit_ct_client_helpers:close_channel(Ch), + rabbit_ct_client_helpers:close_connection(Conn), + passed. + +straight_through(Config) -> + {Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + Q = <<"straight_through-queue">>, + declare(Ch, Q, 3), + [begin + consume(Ch, Q, Ack), + [begin + publish1(Ch, Q, P), + assert_delivered(Ch, Ack, P) + end || P <- [1, 2, 3]], + cancel(Ch) + end || Ack <- [do_ack, no_ack]], + get_empty(Ch, Q), + delete(Ch, Q), + rabbit_ct_client_helpers:close_channel(Ch), + rabbit_ct_client_helpers:close_connection(Conn), + passed. + +max_priorities_above_hard_limit(Config) -> + {Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + Q = <<"max_priorities_above_hard_limit">>, + ?assertExit( + {{shutdown, {server_initiated_close, 406, _}}, _}, + %% Note that lower values (e.g. 300) will overflow the byte type here. + %% However, values >= 256 would still be rejected when used by + %% other clients + declare(Ch, Q, 3000)), + rabbit_ct_client_helpers:close_connection(Conn), + passed. + +negative_max_priorities(Config) -> + {Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + Q = <<"negative_max_priorities">>, + ?assertExit( + {{shutdown, {server_initiated_close, 406, _}}, _}, + declare(Ch, Q, -10)), + rabbit_ct_client_helpers:close_connection(Conn), + passed. + + +invoke(Config) -> + %% Synthetic test to check the invoke callback, as the bug tested here + %% is only triggered with a race condition. + %% When mirroring is stopped, the backing queue of rabbit_amqqueue_process + %% changes from rabbit_mirror_queue_master to rabbit_priority_queue, + %% which shouldn't receive any invoke call. However, there might + %% be pending messages so the priority queue receives the + %% `run_backing_queue` cast message sent to the old master. + A = rabbit_ct_broker_helpers:get_node_config(Config, 0, nodename), + {Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, A), + Q = <<"invoke-queue">>, + declare(Ch, Q, 3), + Pid = queue_pid(Config, A, rabbit_misc:r(<<"/">>, queue, Q)), + rabbit_ct_broker_helpers:rpc( + Config, A, gen_server, cast, + [Pid, + {run_backing_queue, ?MODULE, fun(_, _) -> ok end}]), + Pid2 = queue_pid(Config, A, rabbit_misc:r(<<"/">>, queue, Q)), + Pid = Pid2, + delete(Ch, Q), + rabbit_ct_client_helpers:close_channel(Ch), + rabbit_ct_client_helpers:close_connection(Conn), + passed. + + +gen_server2_stats(Config) -> + %% Synthetic test to check the invoke callback, as the bug tested here + %% is only triggered with a race condition. + %% When mirroring is stopped, the backing queue of rabbit_amqqueue_process + %% changes from rabbit_mirror_queue_master to rabbit_priority_queue, + %% which shouldn't receive any invoke call. However, there might + %% be pending messages so the priority queue receives the + %% `run_backing_queue` cast message sent to the old master. + A = rabbit_ct_broker_helpers:get_node_config(Config, 0, nodename), + {Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, A), + Q = <<"gen_server2_stats_queue">>, + declare(Ch, Q, 3), + Pid = queue_pid(Config, A, rabbit_misc:r(<<"/">>, queue, Q)), + Metrics = rabbit_ct_broker_helpers:rpc( + Config, A, rabbit_core_metrics, get_gen_server2_stats, + [Pid]), + true = is_number(Metrics), + delete(Ch, Q), + rabbit_ct_client_helpers:close_channel(Ch), + rabbit_ct_client_helpers:close_connection(Conn), + passed. + +dropwhile_fetchwhile(Config) -> + {Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + Q = <<"dropwhile_fetchwhile-queue">>, + [begin + declare(Ch, Q, Args ++ arguments(3)), + publish(Ch, Q, [1, 2, 3, 1, 2, 3, 1, 2, 3]), + timer:sleep(10), + get_empty(Ch, Q), + delete(Ch, Q) + end || + Args <- [[{<<"x-message-ttl">>, long, 1}], + [{<<"x-message-ttl">>, long, 1}, + {<<"x-dead-letter-exchange">>, longstr, <<"amq.fanout">>}] + ]], + rabbit_ct_client_helpers:close_channel(Ch), + rabbit_ct_client_helpers:close_connection(Conn), + passed. + +ackfold(Config) -> + {Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + Q = <<"ackfolq-queue1">>, + Q2 = <<"ackfold-queue2">>, + declare(Ch, Q, + [{<<"x-dead-letter-exchange">>, longstr, <<>>}, + {<<"x-dead-letter-routing-key">>, longstr, Q2} + | arguments(3)]), + declare(Ch, Q2, none), + publish(Ch, Q, [1, 2, 3]), + [_, _, DTag] = get_all(Ch, Q, manual_ack, [3, 2, 1]), + amqp_channel:cast(Ch, #'basic.nack'{delivery_tag = DTag, + multiple = true, + requeue = false}), + timer:sleep(100), + get_all(Ch, Q2, do_ack, [3, 2, 1]), + delete(Ch, Q), + delete(Ch, Q2), + rabbit_ct_client_helpers:close_channel(Ch), + rabbit_ct_client_helpers:close_connection(Conn), + passed. + +requeue(Config) -> + {Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + Q = <<"requeue-queue">>, + declare(Ch, Q, 3), + publish(Ch, Q, [1, 2, 3]), + [_, _, DTag] = get_all(Ch, Q, manual_ack, [3, 2, 1]), + amqp_channel:cast(Ch, #'basic.nack'{delivery_tag = DTag, + multiple = true, + requeue = true}), + get_all(Ch, Q, do_ack, [3, 2, 1]), + delete(Ch, Q), + rabbit_ct_client_helpers:close_channel(Ch), + rabbit_ct_client_helpers:close_connection(Conn), + passed. + +drop(Config) -> + {Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + Q = <<"drop-queue">>, + declare(Ch, Q, [{<<"x-max-length">>, long, 4} | arguments(3)]), + publish(Ch, Q, [1, 2, 3, 1, 2, 3, 1, 2, 3]), + %% We drop from the head, so this is according to the "spec" even + %% if not likely to be what the user wants. + get_all(Ch, Q, do_ack, [2, 1, 1, 1]), + delete(Ch, Q), + rabbit_ct_client_helpers:close_channel(Ch), + rabbit_ct_client_helpers:close_connection(Conn), + passed. + +reject(Config) -> + {Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + XOverflow = ?config(overflow, Config), + Q = <<"reject-queue-", XOverflow/binary>>, + declare(Ch, Q, [{<<"x-max-length">>, long, 4}, + {<<"x-overflow">>, longstr, XOverflow} + | arguments(3)]), + publish(Ch, Q, [1, 2, 3, 1, 2, 3, 1, 2, 3]), + %% First 4 messages are published, all others are discarded. + get_all(Ch, Q, do_ack, [3, 2, 1, 1]), + delete(Ch, Q), + rabbit_ct_client_helpers:close_channel(Ch), + rabbit_ct_client_helpers:close_connection(Conn), + passed. + +purge(Config) -> + {Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + Q = <<"purge-queue">>, + declare(Ch, Q, 3), + publish(Ch, Q, [1, 2, 3]), + amqp_channel:call(Ch, #'queue.purge'{queue = Q}), + get_empty(Ch, Q), + delete(Ch, Q), + rabbit_ct_client_helpers:close_channel(Ch), + rabbit_ct_client_helpers:close_connection(Conn), + passed. + +info_head_message_timestamp(Config) -> + passed = rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, info_head_message_timestamp1, [Config]). + +info_head_message_timestamp1(_Config) -> + QName = rabbit_misc:r(<<"/">>, queue, + <<"info_head_message_timestamp-queue">>), + Q0 = rabbit_amqqueue:pseudo_queue(QName, self()), + Q1 = amqqueue:set_arguments(Q0, [{<<"x-max-priority">>, long, 2}]), + PQ = rabbit_priority_queue, + BQS1 = PQ:init(Q1, new, fun(_, _) -> ok end), + %% The queue is empty: no timestamp. + true = PQ:is_empty(BQS1), + '' = PQ:info(head_message_timestamp, BQS1), + %% Publish one message with timestamp 1000. + Msg1 = #basic_message{ + id = msg1, + content = #content{ + properties = #'P_basic'{ + priority = 1, + timestamp = 1000 + }}, + is_persistent = false + }, + BQS2 = PQ:publish(Msg1, #message_properties{size = 0}, false, self(), + noflow, BQS1), + 1000 = PQ:info(head_message_timestamp, BQS2), + %% Publish a higher priority message with no timestamp. + Msg2 = #basic_message{ + id = msg2, + content = #content{ + properties = #'P_basic'{ + priority = 2 + }}, + is_persistent = false + }, + BQS3 = PQ:publish(Msg2, #message_properties{size = 0}, false, self(), + noflow, BQS2), + '' = PQ:info(head_message_timestamp, BQS3), + %% Consume message with no timestamp. + {{Msg2, _, _}, BQS4} = PQ:fetch(false, BQS3), + 1000 = PQ:info(head_message_timestamp, BQS4), + %% Consume message with timestamp 1000, but do not acknowledge it + %% yet. The goal is to verify that the unacknowledged message's + %% timestamp is returned. + {{Msg1, _, AckTag}, BQS5} = PQ:fetch(true, BQS4), + 1000 = PQ:info(head_message_timestamp, BQS5), + %% Ack message. The queue is empty now. + {[msg1], BQS6} = PQ:ack([AckTag], BQS5), + true = PQ:is_empty(BQS6), + '' = PQ:info(head_message_timestamp, BQS6), + PQ:delete_and_terminate(a_whim, BQS6), + passed. + +ram_duration(_Config) -> + QName = rabbit_misc:r(<<"/">>, queue, <<"ram_duration-queue">>), + Q0 = rabbit_amqqueue:pseudo_queue(QName, self()), + Q1 = amqqueue:set_arguments(Q0, [{<<"x-max-priority">>, long, 5}]), + PQ = rabbit_priority_queue, + BQS1 = PQ:init(Q1, new, fun(_, _) -> ok end), + {_Duration1, BQS2} = PQ:ram_duration(BQS1), + BQS3 = PQ:set_ram_duration_target(infinity, BQS2), + BQS4 = PQ:set_ram_duration_target(1, BQS3), + {_Duration2, BQS5} = PQ:ram_duration(BQS4), + PQ:delete_and_terminate(a_whim, BQS5), + passed. + +mirror_queue_sync(Config) -> + {Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + Q = <<"mirror_queue_sync-queue">>, + declare(Ch, Q, 3), + publish(Ch, Q, [1, 2, 3]), + ok = rabbit_ct_broker_helpers:set_ha_policy(Config, 0, + <<"^mirror_queue_sync-queue$">>, <<"all">>), + publish(Ch, Q, [1, 2, 3, 1, 2, 3]), + %% master now has 9, mirror 6. + get_partial(Ch, Q, manual_ack, [3, 3, 3, 2, 2, 2]), + %% So some but not all are unacked at the mirror + Nodename0 = rabbit_ct_broker_helpers:get_node_config(Config, 0, nodename), + rabbit_ct_broker_helpers:control_action(sync_queue, Nodename0, + [binary_to_list(Q)], [{"-p", "/"}]), + wait_for_sync(Config, Nodename0, rabbit_misc:r(<<"/">>, queue, Q)), + rabbit_ct_client_helpers:close_connection(Conn), + passed. + +mirror_queue_sync_priority_above_max(Config) -> + A = rabbit_ct_broker_helpers:get_node_config(Config, 0, nodename), + %% Tests synchronisation of mirrors when priority is higher than max priority. + %% This causes an infinity loop (and test timeout) before rabbitmq-server-795 + {Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, A), + Q = <<"mirror_queue_sync_priority_above_max-queue">>, + declare(Ch, Q, 3), + publish(Ch, Q, [5, 5, 5]), + ok = rabbit_ct_broker_helpers:set_ha_policy(Config, A, + <<".*">>, <<"all">>), + rabbit_ct_broker_helpers:control_action(sync_queue, A, + [binary_to_list(Q)], [{"-p", "/"}]), + wait_for_sync(Config, A, rabbit_misc:r(<<"/">>, queue, Q)), + delete(Ch, Q), + rabbit_ct_client_helpers:close_connection(Conn), + passed. + +mirror_queue_sync_priority_above_max_pending_ack(Config) -> + [A, B] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + %% Tests synchronisation of mirrors when priority is higher than max priority + %% and there are pending acks. + %% This causes an infinity loop (and test timeout) before rabbitmq-server-795 + {Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, A), + Q = <<"mirror_queue_sync_priority_above_max_pending_ack-queue">>, + declare(Ch, Q, 3), + publish(Ch, Q, [5, 5, 5]), + %% Consume but 'forget' to acknowledge + get_without_ack(Ch, Q), + get_without_ack(Ch, Q), + ok = rabbit_ct_broker_helpers:set_ha_policy(Config, A, + <<".*">>, <<"all">>), + rabbit_ct_broker_helpers:control_action(sync_queue, A, + [binary_to_list(Q)], [{"-p", "/"}]), + wait_for_sync(Config, A, rabbit_misc:r(<<"/">>, queue, Q)), + synced_msgs(Config, A, rabbit_misc:r(<<"/">>, queue, Q), 3), + synced_msgs(Config, B, rabbit_misc:r(<<"/">>, queue, Q), 3), + delete(Ch, Q), + rabbit_ct_client_helpers:close_connection(Conn), + passed. + +mirror_queue_auto_ack(Config) -> + A = rabbit_ct_broker_helpers:get_node_config(Config, 0, nodename), + %% Check correct use of AckRequired in the notifications to the mirrors. + %% If mirrors are notified with AckRequired == true when it is false, + %% the mirrors will crash with the depth notification as they will not + %% match the master delta. + %% Bug rabbitmq-server 687 + {Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, A), + Q = <<"mirror_queue_auto_ack-queue">>, + declare(Ch, Q, 3), + publish(Ch, Q, [1, 2, 3]), + ok = rabbit_ct_broker_helpers:set_ha_policy(Config, A, + <<".*">>, <<"all">>), + get_partial(Ch, Q, no_ack, [3, 2, 1]), + + %% Retrieve mirrors + SPids = slave_pids(Config, A, rabbit_misc:r(<<"/">>, queue, Q)), + [{SNode1, _SPid1}, {SNode2, SPid2}] = nodes_and_pids(SPids), + + %% Restart one of the mirrors so `request_depth` is triggered + rabbit_ct_broker_helpers:restart_node(Config, SNode1), + + %% The alive mirror must have the same pid after its neighbour is restarted + timer:sleep(3000), %% ugly but we can't know when the `depth` instruction arrives + Slaves = nodes_and_pids(slave_pids(Config, A, rabbit_misc:r(<<"/">>, queue, Q))), + SPid2 = proplists:get_value(SNode2, Slaves), + + delete(Ch, Q), + rabbit_ct_client_helpers:close_channel(Ch), + rabbit_ct_client_helpers:close_connection(Conn), + passed. + +mirror_queue_sync_order(Config) -> + A = rabbit_ct_broker_helpers:get_node_config(Config, 0, nodename), + B = rabbit_ct_broker_helpers:get_node_config(Config, 1, nodename), + {Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, A), + {Conn2, Ch2} = rabbit_ct_client_helpers:open_connection_and_channel(Config, B), + Q = <<"mirror_queue_sync_order-queue">>, + declare(Ch, Q, 3), + publish_payload(Ch, Q, [{1, <<"msg1">>}, {2, <<"msg2">>}, + {2, <<"msg3">>}, {2, <<"msg4">>}, + {3, <<"msg5">>}]), + rabbit_ct_client_helpers:close_channel(Ch), + + %% Add and sync mirror + ok = rabbit_ct_broker_helpers:set_ha_policy( + Config, A, <<"^mirror_queue_sync_order-queue$">>, <<"all">>), + rabbit_ct_broker_helpers:control_action(sync_queue, A, + [binary_to_list(Q)], [{"-p", "/"}]), + wait_for_sync(Config, A, rabbit_misc:r(<<"/">>, queue, Q)), + + %% Stop the master + rabbit_ct_broker_helpers:stop_node(Config, A), + + get_payload(Ch2, Q, do_ack, [<<"msg5">>, <<"msg2">>, <<"msg3">>, + <<"msg4">>, <<"msg1">>]), + + delete(Ch2, Q), + rabbit_ct_broker_helpers:start_node(Config, A), + rabbit_ct_client_helpers:close_connection(Conn), + rabbit_ct_client_helpers:close_connection(Conn2), + passed. + +mirror_reset_policy(Config) -> + %% Gives time to the master to go through all stages. + %% Might eventually trigger some race conditions from #802, + %% although for that I would expect a longer run and higher + %% number of messages in the system. + mirror_reset_policy(Config, 5000). + +mirror_fast_reset_policy(Config) -> + %% This test seems to trigger the bug tested in invoke/1, but it + %% cannot guarantee it will always happen. Thus, both tests + %% should stay in the test suite. + mirror_reset_policy(Config, 5). + + +mirror_reset_policy(Config, Wait) -> + A = rabbit_ct_broker_helpers:get_node_config(Config, 0, nodename), + {Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, A), + Q = <<"mirror_reset_policy-queue">>, + declare(Ch, Q, 5), + Pid = queue_pid(Config, A, rabbit_misc:r(<<"/">>, queue, Q)), + publish_many(Ch, Q, 20000), + [begin + rabbit_ct_broker_helpers:set_ha_policy( + Config, A, <<"^mirror_reset_policy-queue$">>, <<"all">>, + [{<<"ha-sync-mode">>, <<"automatic">>}]), + timer:sleep(Wait), + rabbit_ct_broker_helpers:clear_policy( + Config, A, <<"^mirror_reset_policy-queue$">>), + timer:sleep(Wait) + end || _ <- lists:seq(1, 10)], + timer:sleep(1000), + ok = rabbit_ct_broker_helpers:set_ha_policy( + Config, A, <<"^mirror_reset_policy-queue$">>, <<"all">>, + [{<<"ha-sync-mode">>, <<"automatic">>}]), + wait_for_sync(Config, A, rabbit_misc:r(<<"/">>, queue, Q), 2), + %% Verify master has not crashed + Pid = queue_pid(Config, A, rabbit_misc:r(<<"/">>, queue, Q)), + delete(Ch, Q), + + rabbit_ct_client_helpers:close_connection(Conn), + passed. + +mirror_stop_pending_followers(Config) -> + A = rabbit_ct_broker_helpers:get_node_config(Config, 0, nodename), + B = rabbit_ct_broker_helpers:get_node_config(Config, 1, nodename), + C = rabbit_ct_broker_helpers:get_node_config(Config, 2, nodename), + + [ok = rabbit_ct_broker_helpers:rpc( + Config, Nodename, application, set_env, [rabbit, slave_wait_timeout, 0]) || Nodename <- [A, B, C]], + + {Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, A), + Q = <<"mirror_stop_pending_followers-queue">>, + declare(Ch, Q, 5), + publish_many(Ch, Q, 20000), + + [begin + rabbit_ct_broker_helpers:set_ha_policy( + Config, A, <<"^mirror_stop_pending_followers-queue$">>, <<"all">>, + [{<<"ha-sync-mode">>, <<"automatic">>}]), + wait_for_sync(Config, A, rabbit_misc:r(<<"/">>, queue, Q), 2), + rabbit_ct_broker_helpers:clear_policy( + Config, A, <<"^mirror_stop_pending_followers-queue$">>) + end || _ <- lists:seq(1, 15)], + + delete(Ch, Q), + + [ok = rabbit_ct_broker_helpers:rpc( + Config, Nodename, application, set_env, [rabbit, slave_wait_timeout, 15000]) || Nodename <- [A, B, C]], + + rabbit_ct_client_helpers:close_connection(Conn), + passed. + +%%---------------------------------------------------------------------------- + +declare(Ch, Q, Args) when is_list(Args) -> + amqp_channel:call(Ch, #'queue.declare'{queue = Q, + durable = true, + arguments = Args}); +declare(Ch, Q, Max) -> + declare(Ch, Q, arguments(Max)). + +delete(Ch, Q) -> + amqp_channel:call(Ch, #'queue.delete'{queue = Q}). + +publish(Ch, Q, Ps) -> + amqp_channel:call(Ch, #'confirm.select'{}), + [publish1(Ch, Q, P) || P <- Ps], + amqp_channel:wait_for_confirms(Ch). + +publish_payload(Ch, Q, PPds) -> + amqp_channel:call(Ch, #'confirm.select'{}), + [publish1(Ch, Q, P, Pd) || {P, Pd} <- PPds], + amqp_channel:wait_for_confirms(Ch). + +publish_many(_Ch, _Q, 0) -> ok; +publish_many( Ch, Q, N) -> publish1(Ch, Q, rand:uniform(5)), + publish_many(Ch, Q, N - 1). + +publish1(Ch, Q, P) -> + amqp_channel:cast(Ch, #'basic.publish'{routing_key = Q}, + #amqp_msg{props = props(P), + payload = priority2bin(P)}). + +publish1(Ch, Q, P, Pd) -> + amqp_channel:cast(Ch, #'basic.publish'{routing_key = Q}, + #amqp_msg{props = props(P), + payload = Pd}). + +props(undefined) -> #'P_basic'{delivery_mode = 2}; +props(P) -> #'P_basic'{priority = P, + delivery_mode = 2}. + +consume(Ch, Q, Ack) -> + amqp_channel:subscribe(Ch, #'basic.consume'{queue = Q, + no_ack = Ack =:= no_ack, + consumer_tag = <<"ctag">>}, + self()), + receive + #'basic.consume_ok'{consumer_tag = <<"ctag">>} -> + ok + end. + +cancel(Ch) -> + amqp_channel:call(Ch, #'basic.cancel'{consumer_tag = <<"ctag">>}). + +assert_delivered(Ch, Ack, P) -> + PBin = priority2bin(P), + receive + {#'basic.deliver'{delivery_tag = DTag}, #amqp_msg{payload = PBin2}} -> + PBin = PBin2, + maybe_ack(Ch, Ack, DTag) + end. + +get_all(Ch, Q, Ack, Ps) -> + DTags = get_partial(Ch, Q, Ack, Ps), + get_empty(Ch, Q), + DTags. + +get_partial(Ch, Q, Ack, Ps) -> + [get_ok(Ch, Q, Ack, priority2bin(P)) || P <- Ps]. + +get_empty(Ch, Q) -> + #'basic.get_empty'{} = amqp_channel:call(Ch, #'basic.get'{queue = Q}). + +get_ok(Ch, Q, Ack, PBin) -> + {#'basic.get_ok'{delivery_tag = DTag}, #amqp_msg{payload = PBin2}} = + amqp_channel:call(Ch, #'basic.get'{queue = Q, + no_ack = Ack =:= no_ack}), + ?assertEqual(PBin, PBin2), + maybe_ack(Ch, Ack, DTag). + +get_payload(Ch, Q, Ack, Ps) -> + [get_ok(Ch, Q, Ack, P) || P <- Ps]. + +get_without_ack(Ch, Q) -> + {#'basic.get_ok'{}, _} = + amqp_channel:call(Ch, #'basic.get'{queue = Q, no_ack = false}). + +maybe_ack(Ch, do_ack, DTag) -> + amqp_channel:cast(Ch, #'basic.ack'{delivery_tag = DTag}), + DTag; +maybe_ack(_Ch, _, DTag) -> + DTag. + +arguments(none) -> []; +arguments(Max) -> [{<<"x-max-priority">>, byte, Max}]. + +priority2bin(undefined) -> <<"undefined">>; +priority2bin(Int) -> list_to_binary(integer_to_list(Int)). + +%%---------------------------------------------------------------------------- + +wait_for_sync(Config, Nodename, Q) -> + wait_for_sync(Config, Nodename, Q, 1). + +wait_for_sync(Config, Nodename, Q, Nodes) -> + wait_for_sync(Config, Nodename, Q, Nodes, 600). + +wait_for_sync(_, _, _, _, 0) -> + throw(sync_timeout); +wait_for_sync(Config, Nodename, Q, Nodes, N) -> + case synced(Config, Nodename, Q, Nodes) of + true -> ok; + false -> timer:sleep(100), + wait_for_sync(Config, Nodename, Q, Nodes, N-1) + end. + +synced(Config, Nodename, Q, Nodes) -> + Info = rabbit_ct_broker_helpers:rpc(Config, Nodename, + rabbit_amqqueue, info_all, [<<"/">>, [name, synchronised_slave_pids]]), + [SSPids] = [Pids || [{name, Q1}, {synchronised_slave_pids, Pids}] <- Info, + Q =:= Q1], + length(SSPids) =:= Nodes. + +synced_msgs(Config, Nodename, Q, Expected) -> + Info = rabbit_ct_broker_helpers:rpc(Config, Nodename, + rabbit_amqqueue, info_all, [<<"/">>, [name, messages]]), + [M] = [M || [{name, Q1}, {messages, M}] <- Info, Q =:= Q1], + M =:= Expected. + +nodes_and_pids(SPids) -> + lists:zip([node(S) || S <- SPids], SPids). + +slave_pids(Config, Nodename, Q) -> + Info = rabbit_ct_broker_helpers:rpc(Config, Nodename, + rabbit_amqqueue, info_all, [<<"/">>, [name, slave_pids]]), + [SPids] = [SPids || [{name, Q1}, {slave_pids, SPids}] <- Info, + Q =:= Q1], + SPids. + +queue_pid(Config, Nodename, Q) -> + Info = rabbit_ct_broker_helpers:rpc( + Config, Nodename, + rabbit_amqqueue, info_all, [<<"/">>, [name, pid]]), + [Pid] = [P || [{name, Q1}, {pid, P}] <- Info, Q =:= Q1], + Pid. + +%%---------------------------------------------------------------------------- diff --git a/deps/rabbit/test/priority_queue_recovery_SUITE.erl b/deps/rabbit/test/priority_queue_recovery_SUITE.erl new file mode 100644 index 0000000000..9679fb0449 --- /dev/null +++ b/deps/rabbit/test/priority_queue_recovery_SUITE.erl @@ -0,0 +1,144 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2011-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(priority_queue_recovery_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). + +-compile(export_all). + +all() -> + [ + {group, non_parallel_tests} + ]. + +groups() -> + [ + {non_parallel_tests, [], [ + recovery %% Restart RabbitMQ. + ]} + ]. + +%% ------------------------------------------------------------------- +%% Testsuite setup/teardown. +%% ------------------------------------------------------------------- + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(_, Config) -> + Config1 = rabbit_ct_helpers:set_config(Config, [ + {rmq_nodes_count, 2} + ]), + rabbit_ct_helpers:run_steps(Config1, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()). + +end_per_group(_, Config) -> + rabbit_ct_helpers:run_steps(Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()). + +init_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_started(Config, Testcase). + +end_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_finished(Config, Testcase). + +%% ------------------------------------------------------------------- +%% Testcases. +%% ------------------------------------------------------------------- + +recovery(Config) -> + {Conn, Ch} = open(Config), + Q = <<"recovery-queue">>, + declare(Ch, Q, 3), + publish(Ch, Q, [1, 2, 3, 1, 2, 3, 1, 2, 3]), + rabbit_ct_client_helpers:close_channel(Ch), + rabbit_ct_client_helpers:close_connection(Conn), + + rabbit_ct_broker_helpers:restart_broker(Config, 0), + + {Conn2, Ch2} = open(Config, 1), + get_all(Ch2, Q, do_ack, [3, 3, 3, 2, 2, 2, 1, 1, 1]), + delete(Ch2, Q), + rabbit_ct_client_helpers:close_channel(Ch2), + rabbit_ct_client_helpers:close_connection(Conn2), + passed. + + +%%---------------------------------------------------------------------------- + +open(Config) -> + open(Config, 0). + +open(Config, NodeIndex) -> + rabbit_ct_client_helpers:open_connection_and_channel(Config, NodeIndex). + +declare(Ch, Q, Args) when is_list(Args) -> + amqp_channel:call(Ch, #'queue.declare'{queue = Q, + durable = true, + arguments = Args}); +declare(Ch, Q, Max) -> + declare(Ch, Q, arguments(Max)). + +delete(Ch, Q) -> + amqp_channel:call(Ch, #'queue.delete'{queue = Q}). + +publish(Ch, Q, Ps) -> + amqp_channel:call(Ch, #'confirm.select'{}), + [publish1(Ch, Q, P) || P <- Ps], + amqp_channel:wait_for_confirms(Ch). + +publish1(Ch, Q, P) -> + amqp_channel:cast(Ch, #'basic.publish'{routing_key = Q}, + #amqp_msg{props = props(P), + payload = priority2bin(P)}). + +publish1(Ch, Q, P, Pd) -> + amqp_channel:cast(Ch, #'basic.publish'{routing_key = Q}, + #amqp_msg{props = props(P), + payload = Pd}). + +get_all(Ch, Q, Ack, Ps) -> + DTags = get_partial(Ch, Q, Ack, Ps), + get_empty(Ch, Q), + DTags. + +get_partial(Ch, Q, Ack, Ps) -> + [get_ok(Ch, Q, Ack, priority2bin(P)) || P <- Ps]. + +get_empty(Ch, Q) -> + #'basic.get_empty'{} = amqp_channel:call(Ch, #'basic.get'{queue = Q}). + +get_ok(Ch, Q, Ack, PBin) -> + {#'basic.get_ok'{delivery_tag = DTag}, #amqp_msg{payload = PBin2}} = + amqp_channel:call(Ch, #'basic.get'{queue = Q, + no_ack = Ack =:= no_ack}), + PBin = PBin2, + maybe_ack(Ch, Ack, DTag). + +maybe_ack(Ch, do_ack, DTag) -> + amqp_channel:cast(Ch, #'basic.ack'{delivery_tag = DTag}), + DTag; +maybe_ack(_Ch, _, DTag) -> + DTag. + +arguments(none) -> []; +arguments(Max) -> [{<<"x-max-priority">>, byte, Max}]. + +priority2bin(undefined) -> <<"undefined">>; +priority2bin(Int) -> list_to_binary(integer_to_list(Int)). + +props(undefined) -> #'P_basic'{delivery_mode = 2}; +props(P) -> #'P_basic'{priority = P, + delivery_mode = 2}. diff --git a/deps/rabbit/test/product_info_SUITE.erl b/deps/rabbit/test/product_info_SUITE.erl new file mode 100644 index 0000000000..207f9222d1 --- /dev/null +++ b/deps/rabbit/test/product_info_SUITE.erl @@ -0,0 +1,171 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(product_info_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-export([suite/0, + all/0, + groups/0, + init_per_suite/1, + end_per_suite/1, + init_per_group/2, + end_per_group/2, + init_per_testcase/2, + end_per_testcase/2, + + override_product_name_in_conf/1, + override_product_version_in_conf/1, + set_motd_in_conf/1 + ]). + +suite() -> + [{timetrap, {minutes, 5}}]. + +all() -> + [ + {group, parallel} + ]. + +groups() -> + [ + {parallel, [], + [override_product_name_in_conf, + override_product_version_in_conf, + set_motd_in_conf]} + ]. + +%% ------------------------------------------------------------------- +%% Testsuite setup/teardown. +%% ------------------------------------------------------------------- + +init_per_suite(Config) -> + case os:type() of + {unix, _} -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config); + _ -> + {skip, "This testsuite is only relevant on Unix"} + end. + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(_, Config) -> + Config. + +end_per_group(_, Config) -> + Config. + +init_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_started(Config, Testcase), + ClusterSize = 1, + TestNumber = rabbit_ct_helpers:testcase_number(Config, ?MODULE, Testcase), + Config1 = rabbit_ct_helpers:set_config( + Config, + [ + {rmq_nodename_suffix, Testcase}, + {tcp_ports_base, {skip_n_nodes, TestNumber * ClusterSize}} + ]), + Config2 = case Testcase of + override_product_name_in_conf -> + rabbit_ct_helpers:merge_app_env( + Config1, + {rabbit, [{product_name, "MyProduct"}]}); + override_product_version_in_conf -> + rabbit_ct_helpers:merge_app_env( + Config1, + {rabbit, [{product_version, "MyVersion"}]}); + set_motd_in_conf -> + PrivDir = ?config(priv_dir, Config), + MotdFile = filename:join(PrivDir, "motd.txt"), + ok = file:write_file(MotdFile, <<"My MOTD\n">>), + C2 = rabbit_ct_helpers:set_config( + Config1, + {motd_file, MotdFile}), + rabbit_ct_helpers:merge_app_env( + C2, + {rabbit, [{motd_file, MotdFile}]}) + end, + rabbit_ct_helpers:run_steps(Config2, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()). + +end_per_testcase(Testcase, Config) -> + Config1 = rabbit_ct_helpers:run_steps(Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()), + rabbit_ct_helpers:testcase_finished(Config1, Testcase). + +%% ------------------------------------------------------------------- +%% Testcases. +%% ------------------------------------------------------------------- + +override_product_name_in_conf(Config) -> + A = rabbit_ct_broker_helpers:get_node_config(Config, 0, nodename), + ProductName = "MyProduct", + ?assertEqual(ProductName, + rabbit_ct_broker_helpers:rpc( + Config, A, rabbit, product_name, [])), + grep_in_log_file(Config, A, ProductName), + grep_in_stdout(Config, A, ProductName). + +override_product_version_in_conf(Config) -> + A = rabbit_ct_broker_helpers:get_node_config(Config, 0, nodename), + ProductVersion = "MyVersion", + ?assertEqual(ProductVersion, + rabbit_ct_broker_helpers:rpc( + Config, A, rabbit, product_version, [])), + grep_in_log_file(Config, A, ProductVersion), + grep_in_stdout(Config, A, ProductVersion). + +set_motd_in_conf(Config) -> + A = rabbit_ct_broker_helpers:get_node_config(Config, 0, nodename), + MotdFile = ?config(motd_file, Config), + ?assertEqual(MotdFile, + rabbit_ct_broker_helpers:rpc( + Config, A, rabbit, motd_file, [])), + {ok, Motd0} = file:read_file(MotdFile), + Motd = string:trim(Motd0, trailing, [$\r,$\n]), + ?assertEqual(Motd, + rabbit_ct_broker_helpers:rpc( + Config, A, rabbit, motd, [])), + grep_in_log_file(Config, A, Motd), + grep_in_stdout(Config, A, Motd). + +grep_in_log_file(Config, Node, String) -> + [Log | _] = rabbit_ct_broker_helpers:rpc( + Config, Node, rabbit, log_locations, []), + ct:pal(?LOW_IMPORTANCE, "Grepping \"~s\" in ~s", [String, Log]), + %% We try to grep several times, in case the log file was not + %% fsync'd yet (and thus we don't see the content yet). + do_grep_in_log_file(String, Log, 30). + +do_grep_in_log_file(String, Log, Retries) -> + {ok, Content} = file:read_file(Log), + case re:run(Content, ["\\b", String, "\\b"], [{capture, none}]) of + match -> + ok; + nomatch when Retries > 1 -> + timer:sleep(1000), + do_grep_in_log_file(String, Log, Retries - 1); + nomatch -> + throw({failed_to_grep, String, Log, Content}) + end. + +grep_in_stdout(Config, Node, String) -> + [Log | _] = rabbit_ct_broker_helpers:rpc( + Config, Node, rabbit, log_locations, []), + LogDir = filename:dirname(Log), + Stdout = filename:join(LogDir, "startup_log"), + ct:pal(?LOW_IMPORTANCE, "Grepping \"~s\" in ~s", [String, Stdout]), + {ok, Content} = file:read_file(Stdout), + ?assertMatch( + match, + re:run(Content, ["\\b", String, "\\b"], [{capture, none}])). diff --git a/deps/rabbit/test/proxy_protocol_SUITE.erl b/deps/rabbit/test/proxy_protocol_SUITE.erl new file mode 100644 index 0000000000..92c29b6063 --- /dev/null +++ b/deps/rabbit/test/proxy_protocol_SUITE.erl @@ -0,0 +1,91 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2017-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(proxy_protocol_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("rabbit_common/include/rabbit.hrl"). + +-compile(export_all). + +-define(TIMEOUT, 5000). + +all() -> + [ + {group, sequential_tests} + ]. + +groups() -> [ + {sequential_tests, [], [ + proxy_protocol, + proxy_protocol_tls + ]} + ]. + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + Config1 = rabbit_ct_helpers:set_config(Config, [ + {rmq_nodename_suffix, ?MODULE} + ]), + Config2 = rabbit_ct_helpers:merge_app_env(Config1, [ + {rabbit, [ + {proxy_protocol, true} + ]} + ]), + Config3 = rabbit_ct_helpers:set_config(Config2, {rabbitmq_ct_tls_verify, verify_none}), + rabbit_ct_helpers:run_setup_steps(Config3, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()). + +init_per_group(_, Config) -> Config. +end_per_group(_, Config) -> Config. + +init_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_started(Config, Testcase). + +end_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_finished(Config, Testcase). + +proxy_protocol(Config) -> + Port = rabbit_ct_broker_helpers:get_node_config(Config, 0, tcp_port_amqp), + {ok, Socket} = gen_tcp:connect({127,0,0,1}, Port, + [binary, {active, false}, {packet, raw}]), + ok = inet:send(Socket, "PROXY TCP4 192.168.1.1 192.168.1.2 80 81\r\n"), + ok = inet:send(Socket, <<"AMQP", 0, 0, 9, 1>>), + {ok, _Packet} = gen_tcp:recv(Socket, 0, ?TIMEOUT), + ConnectionName = rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, connection_name, []), + match = re:run(ConnectionName, <<"^192.168.1.1:80 ">>, [{capture, none}]), + gen_tcp:close(Socket), + ok. + +proxy_protocol_tls(Config) -> + app_utils:start_applications([asn1, crypto, public_key, ssl]), + Port = rabbit_ct_broker_helpers:get_node_config(Config, 0, tcp_port_amqp_tls), + {ok, Socket} = gen_tcp:connect({127,0,0,1}, Port, + [binary, {active, false}, {packet, raw}]), + ok = inet:send(Socket, "PROXY TCP4 192.168.1.1 192.168.1.2 80 81\r\n"), + {ok, SslSocket} = ssl:connect(Socket, [], ?TIMEOUT), + ok = ssl:send(SslSocket, <<"AMQP", 0, 0, 9, 1>>), + {ok, _Packet} = ssl:recv(SslSocket, 0, ?TIMEOUT), + ConnectionName = rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, connection_name, []), + match = re:run(ConnectionName, <<"^192.168.1.1:80 ">>, [{capture, none}]), + gen_tcp:close(Socket), + ok. + +connection_name() -> + Pids = pg_local:get_members(rabbit_connections), + Pid = lists:nth(1, Pids), + {dictionary, Dict} = process_info(Pid, dictionary), + {process_name, {rabbit_reader, ConnectionName}} = lists:keyfind(process_name, 1, Dict), + ConnectionName. diff --git a/deps/rabbit/test/publisher_confirms_parallel_SUITE.erl b/deps/rabbit/test/publisher_confirms_parallel_SUITE.erl new file mode 100644 index 0000000000..f79fcae3ce --- /dev/null +++ b/deps/rabbit/test/publisher_confirms_parallel_SUITE.erl @@ -0,0 +1,380 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2011-2020 VMware, Inc. or its affiliates. All rights reserved. +%% +-module(publisher_confirms_parallel_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("kernel/include/file.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-compile(export_all). + +-define(TIMEOUT, 60000). + +-import(quorum_queue_utils, [wait_for_messages/2]). + +all() -> + [ + {group, publisher_confirm_tests} + ]. + +groups() -> + PublisherConfirmTests = [publisher_confirms, + publisher_confirms_with_deleted_queue, + confirm_select_ok, + confirm_nowait, + confirm_ack, + confirm_acks, + confirm_mandatory_unroutable, + confirm_unroutable_message], + [ + {publisher_confirm_tests, [], + [ + {classic_queue, [parallel], PublisherConfirmTests ++ [confirm_nack]}, + {mirrored_queue, [parallel], PublisherConfirmTests ++ [confirm_nack]}, + {quorum_queue, [], + [ + {parllel_tests, [parallel], PublisherConfirmTests}, + confirm_minority + ]} + ]} + ]. + +suite() -> + [ + {timetrap, {minutes, 3}} + ]. + +%% ------------------------------------------------------------------- +%% Testsuite setup/teardown. +%% ------------------------------------------------------------------- + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(classic_queue, Config) -> + rabbit_ct_helpers:set_config( + Config, + [{queue_args, [{<<"x-queue-type">>, longstr, <<"classic">>}]}, + {queue_durable, true}]); +init_per_group(quorum_queue, Config) -> + case rabbit_ct_broker_helpers:enable_feature_flag(Config, quorum_queue) of + ok -> + rabbit_ct_helpers:set_config( + Config, + [{queue_args, [{<<"x-queue-type">>, longstr, <<"quorum">>}]}, + {queue_durable, true}]); + Skip -> + Skip + end; +init_per_group(mirrored_queue, Config) -> + rabbit_ct_broker_helpers:set_ha_policy(Config, 0, <<"^max_length.*queue">>, + <<"all">>, [{<<"ha-sync-mode">>, <<"automatic">>}]), + Config1 = rabbit_ct_helpers:set_config( + Config, [{is_mirrored, true}, + {queue_args, [{<<"x-queue-type">>, longstr, <<"classic">>}]}, + {queue_durable, true}]), + rabbit_ct_helpers:run_steps(Config1, []); +init_per_group(Group, Config) -> + case lists:member({group, Group}, all()) of + true -> + ClusterSize = 3, + Config1 = rabbit_ct_helpers:set_config(Config, [ + {rmq_nodename_suffix, Group}, + {rmq_nodes_count, ClusterSize} + ]), + rabbit_ct_helpers:run_steps(Config1, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()); + false -> + Config + end. + +end_per_group(Group, Config) -> + case lists:member({group, Group}, all()) of + true -> + rabbit_ct_helpers:run_steps(Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()); + false -> + Config + end. + +init_per_testcase(Testcase, Config) -> + Group = proplists:get_value(name, ?config(tc_group_properties, Config)), + Q = rabbit_data_coercion:to_binary(io_lib:format("~p_~p", [Group, Testcase])), + Q2 = rabbit_data_coercion:to_binary(io_lib:format("~p_~p_2", [Group, Testcase])), + Config1 = rabbit_ct_helpers:set_config(Config, [{queue_name, Q}, + {queue_name_2, Q2}]), + rabbit_ct_helpers:testcase_started(Config1, Testcase). + +end_per_testcase(Testcase, Config) -> + {_, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + amqp_channel:call(Ch, #'queue.delete'{queue = ?config(queue_name, Config)}), + amqp_channel:call(Ch, #'queue.delete'{queue = ?config(queue_name_2, Config)}), + rabbit_ct_helpers:testcase_finished(Config, Testcase). + +%% To enable confirms, a client sends the confirm.select method +publisher_confirms(Config) -> + {_Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + QName = ?config(queue_name, Config), + declare_queue(Ch, Config, QName), + amqp_channel:call(Ch, #'confirm.select'{}), + amqp_channel:register_confirm_handler(Ch, self()), + publish(Ch, QName, [<<"msg1">>]), + wait_for_messages(Config, [[QName, <<"1">>, <<"1">>, <<"0">>]]), + amqp_channel:wait_for_confirms(Ch, 5), + amqp_channel:unregister_confirm_handler(Ch), + ok. + +publisher_confirms_with_deleted_queue(Config) -> + {_Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + QName = ?config(queue_name, Config), + declare_queue(Ch, Config, QName), + amqp_channel:call(Ch, #'confirm.select'{}), + amqp_channel:register_confirm_handler(Ch, self()), + publish(Ch, QName, [<<"msg1">>]), + amqp_channel:call(Ch, #'queue.delete'{queue = QName}), + amqp_channel:wait_for_confirms_or_die(Ch, 5), + amqp_channel:unregister_confirm_handler(Ch). + +%% Depending on whether no-wait was set or not, the broker may respond with a confirm.select-ok +confirm_select_ok(Config) -> + {_Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + QName = ?config(queue_name, Config), + declare_queue(Ch, Config, QName), + ?assertEqual(#'confirm.select_ok'{}, amqp_channel:call(Ch, #'confirm.select'{nowait = false})). + +confirm_nowait(Config) -> + {_Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + QName = ?config(queue_name, Config), + declare_queue(Ch, Config, QName), + ?assertEqual(ok, amqp_channel:call(Ch, #'confirm.select'{nowait = true})). + +%% The broker then confirms messages as it handles them by sending a basic.ack on the same channel. +%% The delivery-tag field contains the sequence number of the confirmed message. +confirm_ack(Config) -> + %% Ensure we receive an ack and not a nack + {_Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + QName = ?config(queue_name, Config), + declare_queue(Ch, Config, QName), + amqp_channel:call(Ch, #'confirm.select'{}), + amqp_channel:register_confirm_handler(Ch, self()), + publish(Ch, QName, [<<"msg1">>]), + receive + #'basic.ack'{delivery_tag = 1} -> + ok + after 5000 -> + throw(missing_ack) + end. + +%% The broker may also set the multiple field in basic.ack to indicate that all messages up to +%% and including the one with the sequence number have been handled. +confirm_acks(Config) -> + {_Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + QName = ?config(queue_name, Config), + declare_queue(Ch, Config, QName), + amqp_channel:call(Ch, #'confirm.select'{}), + amqp_channel:register_confirm_handler(Ch, self()), + publish(Ch, QName, [<<"msg1">>, <<"msg2">>, <<"msg3">>, <<"msg4">>]), + receive_many(lists:seq(1, 4)). + +%% For unroutable messages, the broker will issue a confirm once the exchange verifies a message +%% won't route to any queue (returns an empty list of queues). +%% If the message is also published as mandatory, the basic.return is sent to the client before +%% basic.ack. +confirm_mandatory_unroutable(Config) -> + {_Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + QName = ?config(queue_name, Config), + amqp_channel:call(Ch, #'confirm.select'{}), + amqp_channel:register_confirm_handler(Ch, self()), + amqp_channel:register_return_handler(Ch, self()), + ok = amqp_channel:call(Ch, #'basic.publish'{routing_key = QName, + mandatory = true}, #amqp_msg{payload = <<"msg1">>}), + receive + {#'basic.return'{}, _} -> + ok + after 5000 -> + throw(missing_return) + end, + receive + #'basic.ack'{delivery_tag = 1} -> + ok + after 5000 -> + throw(missing_ack) + end. + +confirm_unroutable_message(Config) -> + %% Ensure we receive a nack for an unroutable message + {_Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + QName = ?config(queue_name, Config), + amqp_channel:call(Ch, #'confirm.select'{}), + amqp_channel:register_confirm_handler(Ch, self()), + publish(Ch, QName, [<<"msg1">>]), + receive + {#'basic.return'{}, _} -> + throw(unexpected_basic_return); + #'basic.ack'{delivery_tag = 1} -> + ok + after 5000 -> + throw(missing_ack) + end. + +%% In exceptional cases when the broker is unable to handle messages successfully, +%% instead of a basic.ack, the broker will send a basic.nack. +%% basic.nack will only be delivered if an internal error occurs in the Erlang process +%% responsible for a queue. +%% This test crashes the queue before it has time to answer, but it only works for classic +%% queues. On quorum queues the followers will take over and rabbit_fifo_client will resend +%% any pending messages. +confirm_nack(Config) -> + passed = rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, confirm_nack1, [Config]). + +confirm_nack1(Config) -> + {_Writer, _Limiter, Ch} = rabbit_ct_broker_helpers:test_channel(), + ok = rabbit_channel:do(Ch, #'channel.open'{}), + receive #'channel.open_ok'{} -> ok + after ?TIMEOUT -> throw(failed_to_receive_channel_open_ok) + end, + Args = ?config(queue_args, Config), + Durable = ?config(queue_durable, Config), + QName1 = ?config(queue_name, Config), + QName2 = ?config(queue_name_2, Config), + DeclareBindDurableQueue = + fun(QName) -> + rabbit_channel:do(Ch, #'queue.declare'{durable = Durable, + queue = QName, + arguments = Args}), + receive #'queue.declare_ok'{} -> + rabbit_channel:do(Ch, #'queue.bind'{ + queue = QName, + exchange = <<"amq.direct">>, + routing_key = "confirms-magic" }), + receive #'queue.bind_ok'{} -> ok + after ?TIMEOUT -> throw(failed_to_bind_queue) + end + after ?TIMEOUT -> throw(failed_to_declare_queue) + end + end, + %% Declare and bind two queues + DeclareBindDurableQueue(QName1), + DeclareBindDurableQueue(QName2), + %% Get the first one's pid (we'll crash it later) + {ok, Q1} = rabbit_amqqueue:lookup(rabbit_misc:r(<<"/">>, queue, QName1)), + QPid1 = amqqueue:get_pid(Q1), + %% Enable confirms + rabbit_channel:do(Ch, #'confirm.select'{}), + receive + #'confirm.select_ok'{} -> ok + after ?TIMEOUT -> throw(failed_to_enable_confirms) + end, + %% Publish a message + rabbit_channel:do(Ch, #'basic.publish'{exchange = <<"amq.direct">>, + routing_key = "confirms-magic" + }, + rabbit_basic:build_content( + #'P_basic'{delivery_mode = 2}, <<"">>)), + %% We must not kill the queue before the channel has processed the + %% 'publish'. + ok = rabbit_channel:flush(Ch), + %% Crash the queue + QPid1 ! boom, + %% Wait for a nack + receive + #'basic.nack'{} -> ok; + #'basic.ack'{} -> throw(received_ack_instead_of_nack) + after ?TIMEOUT-> throw(did_not_receive_nack) + end, + receive + #'basic.ack'{} -> throw(received_ack_when_none_expected) + after 1000 -> ok + end, + %% Cleanup + unlink(Ch), + ok = rabbit_channel:shutdown(Ch), + passed. + +%% The closest to a nack behaviour that we can get on quorum queues is not answering while +%% the cluster is in minority. Once the cluster recovers, a 'basic.ack' will be issued. +confirm_minority(Config) -> + [_A, B, C] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + {_Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + QName = ?config(queue_name, Config), + declare_queue(Ch, Config, QName), + ok = rabbit_ct_broker_helpers:stop_node(Config, B), + ok = rabbit_ct_broker_helpers:stop_node(Config, C), + amqp_channel:call(Ch, #'confirm.select'{}), + amqp_channel:register_confirm_handler(Ch, self()), + publish(Ch, QName, [<<"msg1">>]), + receive + #'basic.nack'{} -> ok; + #'basic.ack'{} -> throw(unexpected_ack) + after 120000 -> + ok + end, + ok = rabbit_ct_broker_helpers:start_node(Config, B), + publish(Ch, QName, [<<"msg2">>]), + receive + #'basic.nack'{} -> throw(unexpected_nack); + #'basic.ack'{} -> ok + after 60000 -> + throw(missing_ack) + end. + +%%%%%%%%%%%%%%%%%%%%%%%% +%% Test helpers +%%%%%%%%%%%%%%%%%%%%%%%% +declare_queue(Ch, Config, QName) -> + Args = ?config(queue_args, Config), + Durable = ?config(queue_durable, Config), + #'queue.declare_ok'{} = amqp_channel:call(Ch, #'queue.declare'{queue = QName, + arguments = Args, + durable = Durable}). + +publish(Ch, QName, Payloads) -> + [amqp_channel:call(Ch, #'basic.publish'{routing_key = QName}, #amqp_msg{payload = Payload}) + || Payload <- Payloads]. + +publish(Ch, QName, Payloads, Headers) -> + [amqp_channel:call(Ch, #'basic.publish'{routing_key = QName}, + #amqp_msg{payload = Payload, + props = #'P_basic'{headers = Headers}}) + || Payload <- Payloads]. + +consume(Ch, QName, Payloads) -> + [begin + {#'basic.get_ok'{delivery_tag = DTag}, #amqp_msg{payload = Payload}} = + amqp_channel:call(Ch, #'basic.get'{queue = QName}), + DTag + end || Payload <- Payloads]. + +consume_empty(Ch, QName) -> + #'basic.get_empty'{} = amqp_channel:call(Ch, #'basic.get'{queue = QName}). + +sync_mirrors(QName, Config) -> + case ?config(is_mirrored, Config) of + true -> + rabbit_ct_broker_helpers:rabbitmqctl(Config, 0, [<<"sync_queue">>, QName]); + _ -> ok + end. + +receive_many([]) -> + ok; +receive_many(DTags) -> + receive + #'basic.ack'{delivery_tag = DTag, multiple = true} -> + receive_many(DTags -- lists:seq(1, DTag)); + #'basic.ack'{delivery_tag = DTag, multiple = false} -> + receive_many(DTags -- [DTag]) + after 5000 -> + throw(missing_ack) + end. diff --git a/deps/rabbit/test/queue_length_limits_SUITE.erl b/deps/rabbit/test/queue_length_limits_SUITE.erl new file mode 100644 index 0000000000..b86f502869 --- /dev/null +++ b/deps/rabbit/test/queue_length_limits_SUITE.erl @@ -0,0 +1,382 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2011-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(queue_length_limits_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("kernel/include/file.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-compile(export_all). + +-define(TIMEOUT_LIST_OPS_PASS, 5000). +-define(TIMEOUT, 30000). +-define(TIMEOUT_CHANNEL_EXCEPTION, 5000). + +-define(CLEANUP_QUEUE_NAME, <<"cleanup-queue">>). + +all() -> + [ + {group, parallel_tests} + ]. + +groups() -> + MaxLengthTests = [max_length_default, + max_length_bytes_default, + max_length_drop_head, + max_length_bytes_drop_head, + max_length_reject_confirm, + max_length_bytes_reject_confirm, + max_length_drop_publish, + max_length_drop_publish_requeue, + max_length_bytes_drop_publish], + [ + {parallel_tests, [parallel], [ + {max_length_classic, [], MaxLengthTests}, + {max_length_quorum, [], [max_length_default, + max_length_bytes_default] + }, + {max_length_mirrored, [], MaxLengthTests} + ]} + ]. + +suite() -> + [ + {timetrap, {minutes, 3}} + ]. + +%% ------------------------------------------------------------------- +%% Testsuite setup/teardown. +%% ------------------------------------------------------------------- + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(max_length_classic, Config) -> + rabbit_ct_helpers:set_config( + Config, + [{queue_args, [{<<"x-queue-type">>, longstr, <<"classic">>}]}, + {queue_durable, false}]); +init_per_group(max_length_quorum, Config) -> + case rabbit_ct_broker_helpers:enable_feature_flag(Config, quorum_queue) of + ok -> + rabbit_ct_helpers:set_config( + Config, + [{queue_args, [{<<"x-queue-type">>, longstr, <<"quorum">>}]}, + {queue_durable, true}]); + Skip -> + Skip + end; +init_per_group(max_length_mirrored, Config) -> + rabbit_ct_broker_helpers:set_ha_policy(Config, 0, <<"^max_length.*queue">>, + <<"all">>, [{<<"ha-sync-mode">>, <<"automatic">>}]), + Config1 = rabbit_ct_helpers:set_config( + Config, [{is_mirrored, true}, + {queue_args, [{<<"x-queue-type">>, longstr, <<"classic">>}]}, + {queue_durable, false}]), + rabbit_ct_helpers:run_steps(Config1, []); +init_per_group(Group, Config) -> + case lists:member({group, Group}, all()) of + true -> + ClusterSize = 3, + Config1 = rabbit_ct_helpers:set_config(Config, [ + {rmq_nodename_suffix, Group}, + {rmq_nodes_count, ClusterSize} + ]), + rabbit_ct_helpers:run_steps(Config1, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()); + false -> + rabbit_ct_helpers:run_steps(Config, []) + end. + +end_per_group(max_length_mirrored, Config) -> + rabbit_ct_broker_helpers:clear_policy(Config, 0, <<"^max_length.*queue">>), + Config1 = rabbit_ct_helpers:set_config(Config, [{is_mirrored, false}]), + Config1; +end_per_group(queue_max_length, Config) -> + Config; +end_per_group(Group, Config) -> + case lists:member({group, Group}, all()) of + true -> + rabbit_ct_helpers:run_steps(Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()); + false -> + Config + end. + +init_per_testcase(Testcase, Config) -> + Group = proplists:get_value(name, ?config(tc_group_properties, Config)), + Q = rabbit_data_coercion:to_binary(io_lib:format("~p_~p", [Group, Testcase])), + Config1 = rabbit_ct_helpers:set_config(Config, [{queue_name, Q}]), + rabbit_ct_helpers:testcase_started(Config1, Testcase). + +end_per_testcase(Testcase, Config) + when Testcase == max_length_drop_publish; Testcase == max_length_bytes_drop_publish; + Testcase == max_length_drop_publish_requeue; + Testcase == max_length_reject_confirm; Testcase == max_length_bytes_reject_confirm; + Testcase == max_length_drop_head; Testcase == max_length_bytes_drop_head; + Testcase == max_length_default; Testcase == max_length_bytes_default -> + {_, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + amqp_channel:call(Ch, #'queue.delete'{queue = ?config(queue_name, Config)}), + rabbit_ct_client_helpers:close_channels_and_connection(Config, 0), + rabbit_ct_helpers:testcase_finished(Config, Testcase); + +end_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_finished(Config, Testcase). + + +%% ------------------------------------------------------------------- +%% Test cases +%% ------------------------------------------------------------------- + +max_length_bytes_drop_head(Config) -> + max_length_bytes_drop_head(Config, [{<<"x-overflow">>, longstr, <<"drop-head">>}]). + +max_length_bytes_default(Config) -> + max_length_bytes_drop_head(Config, []). + +max_length_bytes_drop_head(Config, ExtraArgs) -> + {_Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + Args = ?config(queue_args, Config), + Durable = ?config(queue_durable, Config), + QName = ?config(queue_name, Config), + + MaxLengthBytesArgs = [{<<"x-max-length-bytes">>, long, 100}], + #'queue.declare_ok'{} = amqp_channel:call(Ch, #'queue.declare'{queue = QName, arguments = MaxLengthBytesArgs ++ Args ++ ExtraArgs, durable = Durable}), + + %% 80 bytes payload + Payload1 = << <<"1">> || _ <- lists:seq(1, 80) >>, + Payload2 = << <<"2">> || _ <- lists:seq(1, 80) >>, + Payload3 = << <<"3">> || _ <- lists:seq(1, 80) >>, + check_max_length_drops_head(Config, QName, Ch, Payload1, Payload2, Payload3). + +max_length_drop_head(Config) -> + max_length_drop_head(Config, [{<<"x-overflow">>, longstr, <<"drop-head">>}]). + +max_length_default(Config) -> + %% Defaults to drop_head + max_length_drop_head(Config, []). + +max_length_drop_head(Config, ExtraArgs) -> + {_Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + Args = ?config(queue_args, Config), + Durable = ?config(queue_durable, Config), + QName = ?config(queue_name, Config), + + MaxLengthArgs = [{<<"x-max-length">>, long, 1}], + #'queue.declare_ok'{} = amqp_channel:call(Ch, #'queue.declare'{queue = QName, arguments = MaxLengthArgs ++ Args ++ ExtraArgs, durable = Durable}), + + check_max_length_drops_head(Config, QName, Ch, <<"1">>, <<"2">>, <<"3">>). + +max_length_reject_confirm(Config) -> + {_Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + Args = ?config(queue_args, Config), + QName = ?config(queue_name, Config), + Durable = ?config(queue_durable, Config), + MaxLengthArgs = [{<<"x-max-length">>, long, 1}], + OverflowArgs = [{<<"x-overflow">>, longstr, <<"reject-publish">>}], + #'queue.declare_ok'{} = amqp_channel:call(Ch, #'queue.declare'{queue = QName, arguments = MaxLengthArgs ++ OverflowArgs ++ Args, durable = Durable}), + #'confirm.select_ok'{} = amqp_channel:call(Ch, #'confirm.select'{}), + check_max_length_drops_publish(Config, QName, Ch, <<"1">>, <<"2">>, <<"3">>), + check_max_length_rejects(Config, QName, Ch, <<"1">>, <<"2">>, <<"3">>). + +max_length_bytes_reject_confirm(Config) -> + {_Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + Args = ?config(queue_args, Config), + QNameBytes = ?config(queue_name, Config), + Durable = ?config(queue_durable, Config), + MaxLengthBytesArgs = [{<<"x-max-length-bytes">>, long, 100}], + OverflowArgs = [{<<"x-overflow">>, longstr, <<"reject-publish">>}], + #'queue.declare_ok'{} = amqp_channel:call(Ch, #'queue.declare'{queue = QNameBytes, arguments = MaxLengthBytesArgs ++ OverflowArgs ++ Args, durable = Durable}), + #'confirm.select_ok'{} = amqp_channel:call(Ch, #'confirm.select'{}), + + %% 80 bytes payload + Payload1 = << <<"1">> || _ <- lists:seq(1, 80) >>, + Payload2 = << <<"2">> || _ <- lists:seq(1, 80) >>, + Payload3 = << <<"3">> || _ <- lists:seq(1, 80) >>, + + check_max_length_drops_publish(Config, QNameBytes, Ch, Payload1, Payload2, Payload3), + check_max_length_rejects(Config, QNameBytes, Ch, Payload1, Payload2, Payload3). + +max_length_drop_publish(Config) -> + {_Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + Args = ?config(queue_args, Config), + Durable = ?config(queue_durable, Config), + QName = ?config(queue_name, Config), + MaxLengthArgs = [{<<"x-max-length">>, long, 1}], + OverflowArgs = [{<<"x-overflow">>, longstr, <<"reject-publish">>}], + #'queue.declare_ok'{} = amqp_channel:call(Ch, #'queue.declare'{queue = QName, arguments = MaxLengthArgs ++ OverflowArgs ++ Args, durable = Durable}), + %% If confirms are not enable, publishes will still be dropped in reject-publish mode. + check_max_length_drops_publish(Config, QName, Ch, <<"1">>, <<"2">>, <<"3">>). + +max_length_drop_publish_requeue(Config) -> + {_Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + Args = ?config(queue_args, Config), + Durable = ?config(queue_durable, Config), + QName = ?config(queue_name, Config), + MaxLengthArgs = [{<<"x-max-length">>, long, 1}], + OverflowArgs = [{<<"x-overflow">>, longstr, <<"reject-publish">>}], + #'queue.declare_ok'{} = amqp_channel:call(Ch, #'queue.declare'{queue = QName, arguments = MaxLengthArgs ++ OverflowArgs ++ Args, durable = Durable}), + %% If confirms are not enable, publishes will still be dropped in reject-publish mode. + check_max_length_requeue(Config, QName, Ch, <<"1">>, <<"2">>). + +max_length_bytes_drop_publish(Config) -> + {_Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + Args = ?config(queue_args, Config), + Durable = ?config(queue_durable, Config), + QNameBytes = ?config(queue_name, Config), + MaxLengthBytesArgs = [{<<"x-max-length-bytes">>, long, 100}], + OverflowArgs = [{<<"x-overflow">>, longstr, <<"reject-publish">>}], + #'queue.declare_ok'{} = amqp_channel:call(Ch, #'queue.declare'{queue = QNameBytes, arguments = MaxLengthBytesArgs ++ OverflowArgs ++ Args, durable = Durable}), + + %% 80 bytes payload + Payload1 = << <<"1">> || _ <- lists:seq(1, 80) >>, + Payload2 = << <<"2">> || _ <- lists:seq(1, 80) >>, + Payload3 = << <<"3">> || _ <- lists:seq(1, 80) >>, + + check_max_length_drops_publish(Config, QNameBytes, Ch, Payload1, Payload2, Payload3). + +%% ------------------------------------------------------------------- +%% Implementation +%% ------------------------------------------------------------------- + +check_max_length_requeue(Config, QName, Ch, Payload1, Payload2) -> + sync_mirrors(QName, Config), + + #'confirm.select_ok'{} = amqp_channel:call(Ch, #'confirm.select'{}), + amqp_channel:register_confirm_handler(Ch, self()), + + #'basic.get_empty'{} = amqp_channel:call(Ch, #'basic.get'{queue = QName}), + %% A single message is published and consumed + amqp_channel:call(Ch, #'basic.publish'{routing_key = QName}, #amqp_msg{payload = Payload1}), + amqp_channel:wait_for_confirms(Ch, 5), + + {#'basic.get_ok'{delivery_tag = DeliveryTag}, + #amqp_msg{payload = Payload1}} = amqp_channel:call(Ch, #'basic.get'{queue = QName}), + #'basic.get_empty'{} = amqp_channel:call(Ch, #'basic.get'{queue = QName}), + + %% Another message is published + amqp_channel:call(Ch, #'basic.publish'{routing_key = QName}, #amqp_msg{payload = Payload2}), + amqp_channel:wait_for_confirms(Ch, 5), + + amqp_channel:cast(Ch, #'basic.nack'{delivery_tag = DeliveryTag, + multiple = false, + requeue = true}), + {#'basic.get_ok'{}, #amqp_msg{payload = Payload1}} = amqp_channel:call(Ch, #'basic.get'{queue = QName}), + {#'basic.get_ok'{}, #amqp_msg{payload = Payload2}} = amqp_channel:call(Ch, #'basic.get'{queue = QName}), + #'basic.get_empty'{} = amqp_channel:call(Ch, #'basic.get'{queue = QName}). + +check_max_length_drops_publish(Config, QName, Ch, Payload1, Payload2, Payload3) -> + sync_mirrors(QName, Config), + + #'confirm.select_ok'{} = amqp_channel:call(Ch, #'confirm.select'{}), + amqp_channel:register_confirm_handler(Ch, self()), + + #'basic.get_empty'{} = amqp_channel:call(Ch, #'basic.get'{queue = QName}), + %% A single message is published and consumed + amqp_channel:call(Ch, #'basic.publish'{routing_key = QName}, #amqp_msg{payload = Payload1}), + amqp_channel:wait_for_confirms(Ch, 5), + + {#'basic.get_ok'{}, #amqp_msg{payload = Payload1}} = amqp_channel:call(Ch, #'basic.get'{queue = QName}), + #'basic.get_empty'{} = amqp_channel:call(Ch, #'basic.get'{queue = QName}), + + %% Message 2 is dropped, message 1 stays + amqp_channel:call(Ch, #'basic.publish'{routing_key = QName}, #amqp_msg{payload = Payload1}), + amqp_channel:call(Ch, #'basic.publish'{routing_key = QName}, #amqp_msg{payload = Payload2}), + amqp_channel:wait_for_confirms(Ch, 5), + {#'basic.get_ok'{}, #amqp_msg{payload = Payload1}} = amqp_channel:call(Ch, #'basic.get'{queue = QName}), + #'basic.get_empty'{} = amqp_channel:call(Ch, #'basic.get'{queue = QName}), + + %% Messages 2 and 3 are dropped, message 1 stays + amqp_channel:call(Ch, #'basic.publish'{routing_key = QName}, #amqp_msg{payload = Payload1}), + amqp_channel:call(Ch, #'basic.publish'{routing_key = QName}, #amqp_msg{payload = Payload2}), + amqp_channel:call(Ch, #'basic.publish'{routing_key = QName}, #amqp_msg{payload = Payload3}), + amqp_channel:wait_for_confirms(Ch, 5), + {#'basic.get_ok'{}, #amqp_msg{payload = Payload1}} = amqp_channel:call(Ch, #'basic.get'{queue = QName}), + #'basic.get_empty'{} = amqp_channel:call(Ch, #'basic.get'{queue = QName}). + +check_max_length_rejects(Config, QName, Ch, Payload1, Payload2, Payload3) -> + sync_mirrors(QName, Config), + amqp_channel:register_confirm_handler(Ch, self()), + flush(), + #'basic.get_empty'{} = amqp_channel:call(Ch, #'basic.get'{queue = QName}), + %% First message can be enqueued and acks + amqp_channel:call(Ch, #'basic.publish'{routing_key = QName}, #amqp_msg{payload = Payload1}), + receive #'basic.ack'{} -> ok + after 1000 -> error(expected_ack) + end, + + %% The message cannot be enqueued and nacks + amqp_channel:call(Ch, #'basic.publish'{routing_key = QName}, #amqp_msg{payload = Payload2}), + receive #'basic.nack'{} -> ok + after 1000 -> error(expected_nack) + end, + + %% The message cannot be enqueued and nacks + amqp_channel:call(Ch, #'basic.publish'{routing_key = QName}, #amqp_msg{payload = Payload3}), + receive #'basic.nack'{} -> ok + after 1000 -> error(expected_nack) + end, + + {#'basic.get_ok'{}, #amqp_msg{payload = Payload1}} = amqp_channel:call(Ch, #'basic.get'{queue = QName}), + + %% Now we can publish message 2. + amqp_channel:call(Ch, #'basic.publish'{routing_key = QName}, #amqp_msg{payload = Payload2}), + receive #'basic.ack'{} -> ok + after 1000 -> error(expected_ack) + end, + + {#'basic.get_ok'{}, #amqp_msg{payload = Payload2}} = amqp_channel:call(Ch, #'basic.get'{queue = QName}). + +check_max_length_drops_head(Config, QName, Ch, Payload1, Payload2, Payload3) -> + sync_mirrors(QName, Config), + + #'confirm.select_ok'{} = amqp_channel:call(Ch, #'confirm.select'{}), + amqp_channel:register_confirm_handler(Ch, self()), + + #'basic.get_empty'{} = amqp_channel:call(Ch, #'basic.get'{queue = QName}), + %% A single message is published and consumed + amqp_channel:call(Ch, #'basic.publish'{routing_key = QName}, #amqp_msg{payload = Payload1}), + amqp_channel:wait_for_confirms(Ch, 5), + + {#'basic.get_ok'{}, #amqp_msg{payload = Payload1}} = amqp_channel:call(Ch, #'basic.get'{queue = QName}), + #'basic.get_empty'{} = amqp_channel:call(Ch, #'basic.get'{queue = QName}), + + %% Message 1 is replaced by message 2 + amqp_channel:call(Ch, #'basic.publish'{routing_key = QName}, #amqp_msg{payload = Payload1}), + amqp_channel:call(Ch, #'basic.publish'{routing_key = QName}, #amqp_msg{payload = Payload2}), + amqp_channel:wait_for_confirms(Ch, 5), + + {#'basic.get_ok'{}, #amqp_msg{payload = Payload2}} = amqp_channel:call(Ch, #'basic.get'{queue = QName}), + #'basic.get_empty'{} = amqp_channel:call(Ch, #'basic.get'{queue = QName}), + + %% Messages 1 and 2 are replaced + amqp_channel:call(Ch, #'basic.publish'{routing_key = QName}, #amqp_msg{payload = Payload1}), + amqp_channel:call(Ch, #'basic.publish'{routing_key = QName}, #amqp_msg{payload = Payload2}), + amqp_channel:call(Ch, #'basic.publish'{routing_key = QName}, #amqp_msg{payload = Payload3}), + amqp_channel:wait_for_confirms(Ch, 5), + {#'basic.get_ok'{}, #amqp_msg{payload = Payload3}} = amqp_channel:call(Ch, #'basic.get'{queue = QName}), + #'basic.get_empty'{} = amqp_channel:call(Ch, #'basic.get'{queue = QName}). + +sync_mirrors(QName, Config) -> + case rabbit_ct_helpers:get_config(Config, is_mirrored) of + true -> + rabbit_ct_broker_helpers:rabbitmqctl(Config, 0, [<<"sync_queue">>, QName]); + _ -> ok + end. + +flush() -> + receive _ -> flush() + after 10 -> ok + end. diff --git a/deps/rabbit/test/queue_master_location_SUITE.erl b/deps/rabbit/test/queue_master_location_SUITE.erl new file mode 100644 index 0000000000..fab3eac3f0 --- /dev/null +++ b/deps/rabbit/test/queue_master_location_SUITE.erl @@ -0,0 +1,487 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2011-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(queue_master_location_SUITE). + +%% These tests use an ABC cluster with each node initialised with +%% a different number of queues. When a queue is declared, different +%% strategies can be applied to determine the queue's master node. Queue +%% location strategies can be applied in the following ways; +%% 1. As policy, +%% 2. As config (in rabbitmq.config), +%% 3. or as part of the queue's declare arguments. +%% +%% Currently supported strategies are; +%% min-masters : The queue master node is calculated as the one with the +%% least bound queues in the cluster. +%% client-local: The queue master node is the local node from which +%% the declaration is being carried out from +%% random : The queue master node is randomly selected. +%% + +-include_lib("common_test/include/ct.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-compile(export_all). + +-define(DEFAULT_VHOST_PATH, (<<"/">>)). +-define(POLICY, <<"^qm.location$">>). + +all() -> + [ + {group, cluster_size_3}, + {group, maintenance_mode} + ]. + +groups() -> + [ + {cluster_size_3, [], [ + declare_args, + declare_policy, + declare_invalid_policy, + declare_policy_nodes, + declare_policy_all, + declare_policy_exactly, + declare_config, + calculate_min_master, + calculate_min_master_with_bindings, + calculate_random, + calculate_client_local + ]}, + + {maintenance_mode, [], [ + declare_with_min_masters_and_some_nodes_under_maintenance, + declare_with_min_masters_and_all_nodes_under_maintenance, + + declare_with_random_and_some_nodes_under_maintenance, + declare_with_random_and_all_nodes_under_maintenance + ]} + ]. + +%% ------------------------------------------------------------------- +%% Test suite setup/teardown +%% ------------------------------------------------------------------- + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(cluster_size_3, Config) -> + rabbit_ct_helpers:set_config(Config, [ + %% Replaced with a list of node names later + {rmq_nodes_count, 3} + ]); +init_per_group(maintenance_mode, Config) -> + rabbit_ct_helpers:set_config(Config, [ + {rmq_nodes_count, 3} + ]). + +end_per_group(_, Config) -> + Config. + +init_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_started(Config, Testcase), + ClusterSize = ?config(rmq_nodes_count, Config), + Nodenames = [ + list_to_atom(rabbit_misc:format("~s-~b", [Testcase, I])) + || I <- lists:seq(1, ClusterSize) + ], + TestNumber = rabbit_ct_helpers:testcase_number(Config, ?MODULE, Testcase), + Config1 = rabbit_ct_helpers:set_config(Config, [ + {rmq_nodes_count, Nodenames}, + {rmq_nodes_clustered, true}, + {rmq_nodename_suffix, Testcase}, + {tcp_ports_base, {skip_n_nodes, TestNumber * ClusterSize}} + ]), + Config2 = rabbit_ct_helpers:run_steps( + Config1, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()), + Group = proplists:get_value(name, ?config(tc_group_properties, Config)), + FFEnabled = case Group of + maintenance_mode -> + rabbit_ct_broker_helpers:enable_feature_flag( + Config2, + maintenance_mode_status); + _ -> + ok + end, + case FFEnabled of + ok -> + Config2; + Skip -> + end_per_testcase(Testcase, Config2), + Skip + end. + +end_per_testcase(Testcase, Config) -> + Config1 = rabbit_ct_helpers:run_steps(Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()), + rabbit_ct_helpers:testcase_finished(Config1, Testcase). + +%% ------------------------------------------------------------------- +%% Test cases +%% ------------------------------------------------------------------- + +%% +%% Queue 'declarations' +%% + +declare_args(Config) -> + setup_test_environment(Config), + unset_location_config(Config), + QueueName = rabbit_misc:r(<<"/">>, queue, Q = <<"qm.test">>), + Args = [{<<"x-queue-master-locator">>, longstr, <<"min-masters">>}], + declare(Config, QueueName, false, false, Args, none), + verify_min_master(Config, Q). + +declare_policy(Config) -> + setup_test_environment(Config), + unset_location_config(Config), + set_location_policy(Config, ?POLICY, <<"min-masters">>), + QueueName = rabbit_misc:r(<<"/">>, queue, Q = <<"qm.test">>), + declare(Config, QueueName, false, false, _Args=[], none), + verify_min_master(Config, Q). + +declare_invalid_policy(Config) -> + %% Tests that queue masters location returns 'ok', otherwise the validation of + %% any other parameter might be skipped and invalid policy accepted. + setup_test_environment(Config), + unset_location_config(Config), + Policy = [{<<"queue-master-locator">>, <<"min-masters">>}, + {<<"ha-mode">>, <<"exactly">>}, + %% this field is expected to be an integer + {<<"ha-params">>, <<"2">>}], + {error_string, _} = rabbit_ct_broker_helpers:rpc( + Config, 0, rabbit_policy, set, + [<<"/">>, ?POLICY, <<".*">>, Policy, 0, <<"queues">>, <<"acting-user">>]). + +declare_policy_nodes(Config) -> + setup_test_environment(Config), + unset_location_config(Config), + % Note: + % Node0 has 15 queues, Node1 has 8 and Node2 has 1 + Node0Name = rabbit_data_coercion:to_binary( + rabbit_ct_broker_helpers:get_node_config(Config, 0, nodename)), + Node1 = rabbit_ct_broker_helpers:get_node_config(Config, 1, nodename), + Node1Name = rabbit_data_coercion:to_binary(Node1), + Nodes = [Node1Name, Node0Name], + Policy = [{<<"queue-master-locator">>, <<"min-masters">>}, + {<<"ha-mode">>, <<"nodes">>}, + {<<"ha-params">>, Nodes}], + ok = rabbit_ct_broker_helpers:set_policy(Config, 0, ?POLICY, + <<".*">>, <<"queues">>, Policy), + QueueName = rabbit_misc:r(<<"/">>, queue, Q = <<"qm.test">>), + declare(Config, QueueName, false, false, _Args=[], none), + verify_min_master(Config, Q, Node1). + +declare_policy_all(Config) -> + setup_test_environment(Config), + unset_location_config(Config), + % Note: + % Node0 has 15 queues, Node1 has 8 and Node2 has 1 + Policy = [{<<"queue-master-locator">>, <<"min-masters">>}, + {<<"ha-mode">>, <<"all">>}], + ok = rabbit_ct_broker_helpers:set_policy(Config, 0, ?POLICY, + <<".*">>, <<"queues">>, Policy), + QueueName = rabbit_misc:r(<<"/">>, queue, Q = <<"qm.test">>), + declare(Config, QueueName, false, false, _Args=[], none), + verify_min_master(Config, Q). + +declare_policy_exactly(Config) -> + setup_test_environment(Config), + unset_location_config(Config), + Policy = [{<<"queue-master-locator">>, <<"min-masters">>}, + {<<"ha-mode">>, <<"exactly">>}, + {<<"ha-params">>, 2}], + ok = rabbit_ct_broker_helpers:set_policy(Config, 0, ?POLICY, + <<".*">>, <<"queues">>, Policy), + QueueRes = rabbit_misc:r(<<"/">>, queue, Q = <<"qm.test">>), + declare(Config, QueueRes, false, false, _Args=[], none), + + Node0 = rabbit_ct_broker_helpers:get_node_config(Config, 0, nodename), + rabbit_ct_broker_helpers:control_action(sync_queue, Node0, + [binary_to_list(Q)], [{"-p", "/"}]), + wait_for_sync(Config, Node0, QueueRes, 1), + + {ok, Queue} = rabbit_ct_broker_helpers:rpc(Config, Node0, + rabbit_amqqueue, lookup, [QueueRes]), + {MNode0, [SNode], [SSNode]} = rabbit_ct_broker_helpers:rpc(Config, Node0, + rabbit_mirror_queue_misc, + actual_queue_nodes, [Queue]), + ?assertEqual(SNode, SSNode), + {ok, MNode1} = rabbit_ct_broker_helpers:rpc(Config, 0, + rabbit_queue_master_location_misc, + lookup_master, [Q, ?DEFAULT_VHOST_PATH]), + ?assertEqual(MNode0, MNode1), + Node2 = rabbit_ct_broker_helpers:get_node_config(Config, 2, nodename), + ?assertEqual(MNode1, Node2). + +declare_config(Config) -> + setup_test_environment(Config), + set_location_config(Config, <<"min-masters">>), + QueueName = rabbit_misc:r(<<"/">>, queue, Q = <<"qm.test">>), + declare(Config, QueueName, false, false, _Args = [], none), + verify_min_master(Config, Q), + unset_location_config(Config), + ok. + +%% +%% Maintenance mode effects +%% + +declare_with_min_masters_and_some_nodes_under_maintenance(Config) -> + set_location_policy(Config, ?POLICY, <<"min-masters">>), + rabbit_ct_broker_helpers:mark_as_being_drained(Config, 0), + rabbit_ct_broker_helpers:mark_as_being_drained(Config, 1), + + QName = <<"qm.tests.min_masters.maintenance.case1">>, + Resource = rabbit_misc:r(<<"/">>, queue, QName), + Record = declare(Config, Resource, false, false, _Args = [], none), + %% the only node that's not being drained + ?assertEqual(rabbit_ct_broker_helpers:get_node_config(Config, 2, nodename), + node(amqqueue:get_pid(Record))), + + rabbit_ct_broker_helpers:unmark_as_being_drained(Config, 0), + rabbit_ct_broker_helpers:unmark_as_being_drained(Config, 1). + +declare_with_min_masters_and_all_nodes_under_maintenance(Config) -> + declare_with_all_nodes_under_maintenance(Config, <<"min-masters">>). + +declare_with_random_and_some_nodes_under_maintenance(Config) -> + set_location_policy(Config, ?POLICY, <<"random">>), + rabbit_ct_broker_helpers:mark_as_being_drained(Config, 0), + rabbit_ct_broker_helpers:mark_as_being_drained(Config, 2), + + QName = <<"qm.tests.random.maintenance.case1">>, + Resource = rabbit_misc:r(<<"/">>, queue, QName), + Record = declare(Config, Resource, false, false, _Args = [], none), + %% the only node that's not being drained + ?assertEqual(rabbit_ct_broker_helpers:get_node_config(Config, 1, nodename), + node(amqqueue:get_pid(Record))), + + rabbit_ct_broker_helpers:unmark_as_being_drained(Config, 0), + rabbit_ct_broker_helpers:unmark_as_being_drained(Config, 2). + +declare_with_random_and_all_nodes_under_maintenance(Config) -> + declare_with_all_nodes_under_maintenance(Config, <<"random">>). + +declare_with_all_nodes_under_maintenance(Config, Locator) -> + set_location_policy(Config, ?POLICY, Locator), + rabbit_ct_broker_helpers:mark_as_being_drained(Config, 0), + rabbit_ct_broker_helpers:mark_as_being_drained(Config, 1), + rabbit_ct_broker_helpers:mark_as_being_drained(Config, 2), + + QName = rabbit_data_coercion:to_binary( + rabbit_misc:format("qm.tests.~s.maintenance.case2", [Locator])), + Resource = rabbit_misc:r(<<"/">>, queue, QName), + Record = declare(Config, Resource, false, false, _Args = [], none), + %% when queue master locator returns no node, the node that handles + %% the declaration method will be used as a fallback + ?assertEqual(rabbit_ct_broker_helpers:get_node_config(Config, 0, nodename), + node(amqqueue:get_pid(Record))), + + rabbit_ct_broker_helpers:unmark_as_being_drained(Config, 0), + rabbit_ct_broker_helpers:unmark_as_being_drained(Config, 1), + rabbit_ct_broker_helpers:unmark_as_being_drained(Config, 2). + +%% +%% Test 'calculations' +%% + +calculate_min_master(Config) -> + setup_test_environment(Config), + QueueName = rabbit_misc:r(<<"/">>, queue, Q = <<"qm.test">>), + Args = [{<<"x-queue-master-locator">>, longstr, <<"min-masters">>}], + declare(Config, QueueName, false, false, Args, none), + verify_min_master(Config, Q), + ok. + +calculate_min_master_with_bindings(Config) -> + setup_test_environment(Config), + QueueName = rabbit_misc:r(<<"/">>, queue, Q = <<"qm.test_bound">>), + Args = [{<<"x-queue-master-locator">>, longstr, <<"min-masters">>}], + declare(Config, QueueName, false, false, Args, none), + verify_min_master(Config, Q), + %% Add 20 bindings to this queue + [ bind(Config, QueueName, integer_to_binary(N)) || N <- lists:seq(1, 20) ], + + QueueName1 = rabbit_misc:r(<<"/">>, queue, Q1 = <<"qm.test_unbound">>), + declare(Config, QueueName1, false, false, Args, none), + % Another queue should still be on the same node, bindings should + % not account for min-masters counting + verify_min_master(Config, Q1), + ok. + +calculate_random(Config) -> + setup_test_environment(Config), + QueueName = rabbit_misc:r(<<"/">>, queue, Q = <<"qm.test">>), + Args = [{<<"x-queue-master-locator">>, longstr, <<"random">>}], + declare(Config, QueueName, false, false, Args, none), + verify_random(Config, Q), + ok. + +calculate_client_local(Config) -> + setup_test_environment(Config), + QueueName = rabbit_misc:r(<<"/">>, queue, Q = <<"qm.test">>), + Args = [{<<"x-queue-master-locator">>, longstr, <<"client-local">>}], + declare(Config, QueueName, false, false, Args, none), + verify_client_local(Config, Q), + ok. + +%% +%% Setup environment +%% + +setup_test_environment(Config) -> + Nodes = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + [distribute_queues(Config, Node) || Node <- Nodes], + ok. + +distribute_queues(Config, Node) -> + ok = rpc:call(Node, application, unset_env, [rabbit, queue_master_location]), + Count = case rabbit_ct_broker_helpers:nodename_to_index(Config, Node) of + 0 -> 15; + 1 -> 8; + 2 -> 1 + end, + + Channel = rabbit_ct_client_helpers:open_channel(Config, Node), + ok = declare_queues(Channel, declare_fun(), Count), + ok = create_e2e_binding(Channel, [<< "ex_1" >>, << "ex_2" >>]), + {ok, Channel}. + +%% +%% Internal queue handling +%% + +declare_queues(Channel, DeclareFun, 1) -> DeclareFun(Channel); +declare_queues(Channel, DeclareFun, N) -> + DeclareFun(Channel), + declare_queues(Channel, DeclareFun, N-1). + +declare_exchange(Channel, Ex) -> + #'exchange.declare_ok'{} = + amqp_channel:call(Channel, #'exchange.declare'{exchange = Ex}), + {ok, Ex}. + +declare_binding(Channel, Binding) -> + #'exchange.bind_ok'{} = amqp_channel:call(Channel, Binding), + ok. + +declare_fun() -> + fun(Channel) -> + #'queue.declare_ok'{} = amqp_channel:call(Channel, get_random_queue_declare()), + ok + end. + +create_e2e_binding(Channel, ExNamesBin) -> + [{ok, Ex1}, {ok, Ex2}] = [declare_exchange(Channel, Ex) || Ex <- ExNamesBin], + Binding = #'exchange.bind'{source = Ex1, destination = Ex2}, + ok = declare_binding(Channel, Binding). + +get_random_queue_declare() -> + #'queue.declare'{passive = false, + durable = false, + exclusive = true, + auto_delete = false, + nowait = false, + arguments = []}. + +%% +%% Internal helper functions +%% + +get_cluster() -> [node()|nodes()]. + +min_master_node(Config) -> + hd(lists:reverse( + rabbit_ct_broker_helpers:get_node_configs(Config, nodename))). + +set_location_config(Config, Strategy) -> + Nodes = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + [ok = rabbit_ct_broker_helpers:rpc(Config, Node, + application, set_env, + [rabbit, queue_master_locator, Strategy]) || Node <- Nodes], + ok. + +unset_location_config(Config) -> + Nodes = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + [ok = rabbit_ct_broker_helpers:rpc(Config, Node, + application, unset_env, + [rabbit, queue_master_locator]) || Node <- Nodes], + ok. + +declare(Config, QueueName, Durable, AutoDelete, Args0, Owner) -> + Args1 = [QueueName, Durable, AutoDelete, Args0, Owner, <<"acting-user">>], + case rabbit_ct_broker_helpers:rpc(Config, 0, rabbit_amqqueue, declare, Args1) of + {new, Queue} -> Queue; + Other -> Other + end. + +bind(Config, QueueName, RoutingKey) -> + ExchangeName = rabbit_misc:r(QueueName, exchange, <<"amq.direct">>), + + ok = rabbit_ct_broker_helpers:rpc( + Config, 0, rabbit_binding, add, + [#binding{source = ExchangeName, + destination = QueueName, + key = RoutingKey, + args = []}, + <<"acting-user">>]). + +verify_min_master(Config, Q, MinMasterNode) -> + Rpc = rabbit_ct_broker_helpers:rpc(Config, 0, + rabbit_queue_master_location_misc, + lookup_master, [Q, ?DEFAULT_VHOST_PATH]), + ?assertEqual({ok, MinMasterNode}, Rpc). + +verify_min_master(Config, Q) -> + MinMaster = min_master_node(Config), + verify_min_master(Config, Q, MinMaster). + +verify_random(Config, Q) -> + [Node | _] = Nodes = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + {ok, Master} = rabbit_ct_broker_helpers:rpc(Config, Node, + rabbit_queue_master_location_misc, + lookup_master, [Q, ?DEFAULT_VHOST_PATH]), + ?assert(lists:member(Master, Nodes)). + +verify_client_local(Config, Q) -> + Node = rabbit_ct_broker_helpers:get_node_config(Config, 0, nodename), + Rpc = rabbit_ct_broker_helpers:rpc(Config, Node, + rabbit_queue_master_location_misc, + lookup_master, [Q, ?DEFAULT_VHOST_PATH]), + ?assertEqual({ok, Node}, Rpc). + +set_location_policy(Config, Name, Strategy) -> + ok = rabbit_ct_broker_helpers:set_policy(Config, 0, + Name, <<".*">>, <<"queues">>, [{<<"queue-master-locator">>, Strategy}]). + +wait_for_sync(Config, Nodename, Q, ExpectedSSPidLen) -> + wait_for_sync(Config, Nodename, Q, ExpectedSSPidLen, 600). + +wait_for_sync(_, _, _, _, 0) -> + throw(sync_timeout); +wait_for_sync(Config, Nodename, Q, ExpectedSSPidLen, N) -> + case synced(Config, Nodename, Q, ExpectedSSPidLen) of + true -> ok; + false -> timer:sleep(100), + wait_for_sync(Config, Nodename, Q, ExpectedSSPidLen, N-1) + end. + +synced(Config, Nodename, Q, ExpectedSSPidLen) -> + Args = [<<"/">>, [name, synchronised_slave_pids]], + Info = rabbit_ct_broker_helpers:rpc(Config, Nodename, + rabbit_amqqueue, info_all, Args), + [SSPids] = [Pids || [{name, Q1}, {synchronised_slave_pids, Pids}] <- Info, Q =:= Q1], + length(SSPids) =:= ExpectedSSPidLen. diff --git a/deps/rabbit/test/queue_parallel_SUITE.erl b/deps/rabbit/test/queue_parallel_SUITE.erl new file mode 100644 index 0000000000..6f813512f4 --- /dev/null +++ b/deps/rabbit/test/queue_parallel_SUITE.erl @@ -0,0 +1,725 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2011-2020 VMware, Inc. or its affiliates. All rights reserved. +%% +%% +-module(queue_parallel_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("kernel/include/file.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-compile(export_all). + +-define(TIMEOUT, 30000). + +-import(quorum_queue_utils, [wait_for_messages/2]). + +all() -> + [ + {group, parallel_tests} + ]. + +groups() -> + AllTests = [publish, + consume, + consume_first_empty, + consume_from_empty_queue, + consume_and_autoack, + subscribe, + subscribe_consumers, + subscribe_with_autoack, + consume_and_ack, + consume_and_multiple_ack, + subscribe_and_ack, + subscribe_and_multiple_ack, + subscribe_and_requeue_multiple_nack, + subscribe_and_nack, + subscribe_and_requeue_nack, + subscribe_and_multiple_nack, + consume_and_requeue_nack, + consume_and_nack, + consume_and_requeue_multiple_nack, + consume_and_multiple_nack, + basic_cancel, + purge, + basic_recover, + delete_immediately_by_resource + ], + [ + {parallel_tests, [], + [ + {classic_queue, [parallel], AllTests ++ [delete_immediately_by_pid_succeeds, + trigger_message_store_compaction]}, + {mirrored_queue, [parallel], AllTests ++ [delete_immediately_by_pid_succeeds, + trigger_message_store_compaction]}, + {quorum_queue, [parallel], AllTests ++ [delete_immediately_by_pid_fails]}, + {quorum_queue_in_memory_limit, [parallel], AllTests ++ [delete_immediately_by_pid_fails]}, + {quorum_queue_in_memory_bytes, [parallel], AllTests ++ [delete_immediately_by_pid_fails]}, + {stream_queue, [parallel], [publish, + subscribe]} + ]} + ]. + +suite() -> + [ + {timetrap, {minutes, 3}} + ]. + +%% ------------------------------------------------------------------- +%% Testsuite setup/teardown. +%% ------------------------------------------------------------------- + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(classic_queue, Config) -> + rabbit_ct_helpers:set_config( + Config, + [{queue_args, [{<<"x-queue-type">>, longstr, <<"classic">>}]}, + {consumer_args, []}, + {queue_durable, true}]); +init_per_group(quorum_queue, Config) -> + case rabbit_ct_broker_helpers:enable_feature_flag(Config, quorum_queue) of + ok -> + rabbit_ct_helpers:set_config( + Config, + [{queue_args, [{<<"x-queue-type">>, longstr, <<"quorum">>}]}, + {consumer_args, []}, + {queue_durable, true}]); + Skip -> + Skip + end; +init_per_group(quorum_queue_in_memory_limit, Config) -> + case rabbit_ct_broker_helpers:enable_feature_flag(Config, quorum_queue) of + ok -> + rabbit_ct_helpers:set_config( + Config, + [{queue_args, [{<<"x-queue-type">>, longstr, <<"quorum">>}, + {<<"x-max-in-memory-length">>, long, 1}]}, + {consumer_args, []}, + {queue_durable, true}]); + Skip -> + Skip + end; +init_per_group(quorum_queue_in_memory_bytes, Config) -> + case rabbit_ct_broker_helpers:enable_feature_flag(Config, quorum_queue) of + ok -> + rabbit_ct_helpers:set_config( + Config, + [{queue_args, [{<<"x-queue-type">>, longstr, <<"quorum">>}, + {<<"x-max-in-memory-bytes">>, long, 1}]}, + {consumer_args, []}, + {queue_durable, true}]); + Skip -> + Skip + end; +init_per_group(mirrored_queue, Config) -> + rabbit_ct_broker_helpers:set_ha_policy(Config, 0, <<"^max_length.*queue">>, + <<"all">>, [{<<"ha-sync-mode">>, <<"automatic">>}]), + Config1 = rabbit_ct_helpers:set_config( + Config, [{is_mirrored, true}, + {queue_args, [{<<"x-queue-type">>, longstr, <<"classic">>}]}, + {consumer_args, []}, + {queue_durable, true}]), + rabbit_ct_helpers:run_steps(Config1, []); +init_per_group(stream_queue, Config) -> + case rabbit_ct_broker_helpers:enable_feature_flag(Config, stream_queue) of + ok -> + rabbit_ct_helpers:set_config( + Config, + [{queue_args, [{<<"x-queue-type">>, longstr, <<"stream">>}]}, + {consumer_args, [{<<"x-stream-offset">>, long, 0}]}, + {queue_durable, true}]); + Skip -> + Skip + end; +init_per_group(Group, Config0) -> + case lists:member({group, Group}, all()) of + true -> + ClusterSize = 3, + Config = rabbit_ct_helpers:merge_app_env( + Config0, {rabbit, [{channel_tick_interval, 1000}, + {quorum_tick_interval, 1000}, + {stream_tick_interval, 1000}]}), + Config1 = rabbit_ct_helpers:set_config( + Config, [ {rmq_nodename_suffix, Group}, + {rmq_nodes_count, ClusterSize} + ]), + rabbit_ct_helpers:run_steps(Config1, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()); + false -> + rabbit_ct_helpers:run_steps(Config0, []) + end. + +end_per_group(Group, Config) -> + case lists:member({group, Group}, all()) of + true -> + rabbit_ct_helpers:run_steps(Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()); + false -> + Config + end. + +init_per_testcase(Testcase, Config) -> + Group = proplists:get_value(name, ?config(tc_group_properties, Config)), + Q = rabbit_data_coercion:to_binary(io_lib:format("~p_~p", [Group, Testcase])), + Q2 = rabbit_data_coercion:to_binary(io_lib:format("~p_~p_2", [Group, Testcase])), + Config1 = rabbit_ct_helpers:set_config(Config, [{queue_name, Q}, + {queue_name_2, Q2}]), + rabbit_ct_helpers:testcase_started(Config1, Testcase). + +end_per_testcase(Testcase, Config) -> + {_, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + amqp_channel:call(Ch, #'queue.delete'{queue = ?config(queue_name, Config)}), + amqp_channel:call(Ch, #'queue.delete'{queue = ?config(queue_name_2, Config)}), + rabbit_ct_helpers:testcase_finished(Config, Testcase). + +publish(Config) -> + {_, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + QName = ?config(queue_name, Config), + declare_queue(Ch, Config, QName), + publish(Ch, QName, [<<"msg1">>]), + wait_for_messages(Config, [[QName, <<"1">>, <<"1">>, <<"0">>]]). + +consume(Config) -> + {_, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + QName = ?config(queue_name, Config), + declare_queue(Ch, Config, QName), + publish(Ch, QName, [<<"msg1">>]), + wait_for_messages(Config, [[QName, <<"1">>, <<"1">>, <<"0">>]]), + consume(Ch, QName, [<<"msg1">>]), + wait_for_messages(Config, [[QName, <<"1">>, <<"0">>, <<"1">>]]), + rabbit_ct_client_helpers:close_channel(Ch), + wait_for_messages(Config, [[QName, <<"1">>, <<"1">>, <<"0">>]]). + +consume_first_empty(Config) -> + {_, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + QName = ?config(queue_name, Config), + declare_queue(Ch, Config, QName), + consume_empty(Ch, QName), + publish(Ch, QName, [<<"msg1">>]), + wait_for_messages(Config, [[QName, <<"1">>, <<"1">>, <<"0">>]]), + consume(Ch, QName, true, [<<"msg1">>]), + rabbit_ct_client_helpers:close_channel(Ch). + +consume_from_empty_queue(Config) -> + {_, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + QName = ?config(queue_name, Config), + declare_queue(Ch, Config, QName), + consume_empty(Ch, QName). + +consume_and_autoack(Config) -> + {_, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + QName = ?config(queue_name, Config), + declare_queue(Ch, Config, QName), + publish(Ch, QName, [<<"msg1">>]), + wait_for_messages(Config, [[QName, <<"1">>, <<"1">>, <<"0">>]]), + consume(Ch, QName, true, [<<"msg1">>]), + wait_for_messages(Config, [[QName, <<"0">>, <<"0">>, <<"0">>]]), + rabbit_ct_client_helpers:close_channel(Ch), + wait_for_messages(Config, [[QName, <<"0">>, <<"0">>, <<"0">>]]). + +subscribe(Config) -> + {_, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + QName = ?config(queue_name, Config), + declare_queue(Ch, Config, QName), + + %% Let's set consumer prefetch so it works with stream queues + ?assertMatch(#'basic.qos_ok'{}, + amqp_channel:call(Ch, #'basic.qos'{global = false, + prefetch_count = 10})), + publish(Ch, QName, [<<"msg1">>]), + wait_for_messages(Config, [[QName, <<"1">>, <<"1">>, <<"0">>]]), + + CArgs = ?config(consumer_args, Config), + subscribe(Ch, QName, false, CArgs), + receive_basic_deliver(false), + + rabbit_ct_client_helpers:close_channel(Ch), + wait_for_messages(Config, [[QName, <<"1">>, <<"1">>, <<"0">>]]). + +subscribe_consumers(Config) -> + [Server | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + {_, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + QName = ?config(queue_name, Config), + declare_queue(Ch, Config, QName), + + CArgs = ?config(consumer_args, Config), + ?assertMatch(#'basic.qos_ok'{}, + amqp_channel:call(Ch, #'basic.qos'{global = false, + prefetch_count = 10})), + subscribe(Ch, QName, false, CArgs), + + %% validate we can retrieve the consumers + Consumers = rpc:call(Server, rabbit_amqqueue, consumers_all, [<<"/">>]), + [Consumer] = lists:filter(fun(Props) -> + Resource = proplists:get_value(queue_name, Props), + QName == Resource#resource.name + end, Consumers), + ?assert(is_pid(proplists:get_value(channel_pid, Consumer))), + ?assert(is_binary(proplists:get_value(consumer_tag, Consumer))), + ?assertEqual(true, proplists:get_value(ack_required, Consumer)), + ?assertEqual(10, proplists:get_value(prefetch_count, Consumer)), + ?assertEqual([], proplists:get_value(arguments, Consumer)), + + rabbit_ct_client_helpers:close_channel(Ch). + +subscribe_with_autoack(Config) -> + {_, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + QName = ?config(queue_name, Config), + CArgs = ?config(consumer_args, Config), + declare_queue(Ch, Config, QName), + + publish(Ch, QName, [<<"msg1">>, <<"msg2">>]), + wait_for_messages(Config, [[QName, <<"2">>, <<"2">>, <<"0">>]]), + subscribe(Ch, QName, true, CArgs), + receive_basic_deliver(false), + receive_basic_deliver(false), + wait_for_messages(Config, [[QName, <<"0">>, <<"0">>, <<"0">>]]), + rabbit_ct_client_helpers:close_channel(Ch), + wait_for_messages(Config, [[QName, <<"0">>, <<"0">>, <<"0">>]]). + +consume_and_ack(Config) -> + {_, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + QName = ?config(queue_name, Config), + declare_queue(Ch, Config, QName), + + publish(Ch, QName, [<<"msg1">>]), + wait_for_messages(Config, [[QName, <<"1">>, <<"1">>, <<"0">>]]), + [DeliveryTag] = consume(Ch, QName, [<<"msg1">>]), + wait_for_messages(Config, [[QName, <<"1">>, <<"0">>, <<"1">>]]), + amqp_channel:cast(Ch, #'basic.ack'{delivery_tag = DeliveryTag}), + wait_for_messages(Config, [[QName, <<"0">>, <<"0">>, <<"0">>]]), + rabbit_ct_client_helpers:close_channel(Ch), + ok. + +consume_and_multiple_ack(Config) -> + {_, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + QName = ?config(queue_name, Config), + declare_queue(Ch, Config, QName), + + publish(Ch, QName, [<<"msg1">>, <<"msg2">>, <<"msg3">>]), + wait_for_messages(Config, [[QName, <<"3">>, <<"3">>, <<"0">>]]), + [_, _, DeliveryTag] = consume(Ch, QName, [<<"msg1">>, <<"msg2">>, <<"msg3">>]), + wait_for_messages(Config, [[QName, <<"3">>, <<"0">>, <<"3">>]]), + amqp_channel:cast(Ch, #'basic.ack'{delivery_tag = DeliveryTag, + multiple = true}), + wait_for_messages(Config, [[QName, <<"0">>, <<"0">>, <<"0">>]]), + rabbit_ct_client_helpers:close_channel(Ch), + ok. + +subscribe_and_ack(Config) -> + {_, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + QName = ?config(queue_name, Config), + CArgs = ?config(consumer_args, Config), + declare_queue(Ch, Config, QName), + + publish(Ch, QName, [<<"msg1">>]), + wait_for_messages(Config, [[QName, <<"1">>, <<"1">>, <<"0">>]]), + subscribe(Ch, QName, false, CArgs), + receive + {#'basic.deliver'{delivery_tag = DeliveryTag}, _} -> + wait_for_messages(Config, [[QName, <<"1">>, <<"0">>, <<"1">>]]), + amqp_channel:cast(Ch, #'basic.ack'{delivery_tag = DeliveryTag}), + wait_for_messages(Config, [[QName, <<"0">>, <<"0">>, <<"0">>]]) + end, + rabbit_ct_client_helpers:close_channel(Ch), + ok. + +subscribe_and_multiple_ack(Config) -> + {_, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + QName = ?config(queue_name, Config), + CArgs = ?config(consumer_args, Config), + declare_queue(Ch, Config, QName), + + publish(Ch, QName, [<<"msg1">>, <<"msg2">>, <<"msg3">>]), + wait_for_messages(Config, [[QName, <<"3">>, <<"3">>, <<"0">>]]), + subscribe(Ch, QName, false, CArgs), + receive_basic_deliver(false), + receive_basic_deliver(false), + receive + {#'basic.deliver'{delivery_tag = DeliveryTag}, _} -> + wait_for_messages(Config, [[QName, <<"3">>, <<"0">>, <<"3">>]]), + amqp_channel:cast(Ch, #'basic.ack'{delivery_tag = DeliveryTag, + multiple = true}), + wait_for_messages(Config, [[QName, <<"0">>, <<"0">>, <<"0">>]]) + end, + rabbit_ct_client_helpers:close_channel(Ch), + ok. + +trigger_message_store_compaction(Config) -> + {_, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + QName = ?config(queue_name, Config), + declare_queue(Ch, Config, QName), + + N = 12000, + [publish(Ch, QName, [binary:copy(<<"a">>, 5000)]) || _ <- lists:seq(1, N)], + wait_for_messages(Config, [[QName, <<"12000">>, <<"12000">>, <<"0">>]]), + + AllDTags = rabbit_ct_client_helpers:consume_without_acknowledging(Ch, QName, N), + ToAck = lists:filter(fun (I) -> I > 500 andalso I < 11200 end, AllDTags), + + [amqp_channel:cast(Ch, #'basic.ack'{delivery_tag = Tag, + multiple = false}) || Tag <- ToAck], + + %% give compaction a moment to start in and finish + timer:sleep(5000), + amqp_channel:cast(Ch, #'queue.purge'{queue = QName}), + rabbit_ct_client_helpers:close_channel(Ch), + ok. + +subscribe_and_requeue_multiple_nack(Config) -> + {_, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + QName = ?config(queue_name, Config), + CArgs = ?config(consumer_args, Config), + declare_queue(Ch, Config, QName), + + publish(Ch, QName, [<<"msg1">>, <<"msg2">>, <<"msg3">>]), + wait_for_messages(Config, [[QName, <<"3">>, <<"3">>, <<"0">>]]), + subscribe(Ch, QName, false, CArgs), + receive_basic_deliver(false), + receive_basic_deliver(false), + receive + {#'basic.deliver'{delivery_tag = DeliveryTag, + redelivered = false}, _} -> + wait_for_messages(Config, [[QName, <<"3">>, <<"0">>, <<"3">>]]), + amqp_channel:cast(Ch, #'basic.nack'{delivery_tag = DeliveryTag, + multiple = true, + requeue = true}), + receive_basic_deliver(true), + receive_basic_deliver(true), + receive + {#'basic.deliver'{delivery_tag = DeliveryTag1, + redelivered = true}, _} -> + wait_for_messages(Config, [[QName, <<"3">>, <<"0">>, <<"3">>]]), + amqp_channel:cast(Ch, #'basic.ack'{delivery_tag = DeliveryTag1, + multiple = true}), + wait_for_messages(Config, [[QName, <<"0">>, <<"0">>, <<"0">>]]) + end + end, + rabbit_ct_client_helpers:close_channel(Ch), + ok. + +consume_and_requeue_nack(Config) -> + {_, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + QName = ?config(queue_name, Config), + declare_queue(Ch, Config, QName), + + publish(Ch, QName, [<<"msg1">>, <<"msg2">>]), + wait_for_messages(Config, [[QName, <<"2">>, <<"2">>, <<"0">>]]), + [DeliveryTag] = consume(Ch, QName, [<<"msg1">>]), + wait_for_messages(Config, [[QName, <<"2">>, <<"1">>, <<"1">>]]), + amqp_channel:cast(Ch, #'basic.nack'{delivery_tag = DeliveryTag, + multiple = false, + requeue = true}), + wait_for_messages(Config, [[QName, <<"2">>, <<"2">>, <<"0">>]]), + rabbit_ct_client_helpers:close_channel(Ch), + ok. + +consume_and_nack(Config) -> + {_, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + QName = ?config(queue_name, Config), + declare_queue(Ch, Config, QName), + + publish(Ch, QName, [<<"msg1">>]), + wait_for_messages(Config, [[QName, <<"1">>, <<"1">>, <<"0">>]]), + [DeliveryTag] = consume(Ch, QName, [<<"msg1">>]), + wait_for_messages(Config, [[QName, <<"1">>, <<"0">>, <<"1">>]]), + amqp_channel:cast(Ch, #'basic.nack'{delivery_tag = DeliveryTag, + multiple = false, + requeue = false}), + wait_for_messages(Config, [[QName, <<"0">>, <<"0">>, <<"0">>]]), + rabbit_ct_client_helpers:close_channel(Ch), + ok. + +consume_and_requeue_multiple_nack(Config) -> + {_, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + QName = ?config(queue_name, Config), + declare_queue(Ch, Config, QName), + + publish(Ch, QName, [<<"msg1">>, <<"msg2">>, <<"msg3">>]), + wait_for_messages(Config, [[QName, <<"3">>, <<"3">>, <<"0">>]]), + [_, _, DeliveryTag] = consume(Ch, QName, [<<"msg1">>, <<"msg2">>, <<"msg3">>]), + wait_for_messages(Config, [[QName, <<"3">>, <<"0">>, <<"3">>]]), + amqp_channel:cast(Ch, #'basic.nack'{delivery_tag = DeliveryTag, + multiple = true, + requeue = true}), + wait_for_messages(Config, [[QName, <<"3">>, <<"3">>, <<"0">>]]), + rabbit_ct_client_helpers:close_channel(Ch), + ok. + +consume_and_multiple_nack(Config) -> + {_, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + QName = ?config(queue_name, Config), + declare_queue(Ch, Config, QName), + + publish(Ch, QName, [<<"msg1">>, <<"msg2">>, <<"msg3">>]), + wait_for_messages(Config, [[QName, <<"3">>, <<"3">>, <<"0">>]]), + [_, _, DeliveryTag] = consume(Ch, QName, [<<"msg1">>, <<"msg2">>, <<"msg3">>]), + wait_for_messages(Config, [[QName, <<"3">>, <<"0">>, <<"3">>]]), + amqp_channel:cast(Ch, #'basic.nack'{delivery_tag = DeliveryTag, + multiple = true, + requeue = false}), + wait_for_messages(Config, [[QName, <<"0">>, <<"0">>, <<"0">>]]), + rabbit_ct_client_helpers:close_channel(Ch), + ok. + +subscribe_and_requeue_nack(Config) -> + {_, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + QName = ?config(queue_name, Config), + CArgs = ?config(consumer_args, Config), + declare_queue(Ch, Config, QName), + + publish(Ch, QName, [<<"msg1">>]), + wait_for_messages(Config, [[QName, <<"1">>, <<"1">>, <<"0">>]]), + subscribe(Ch, QName, false, CArgs), + receive + {#'basic.deliver'{delivery_tag = DeliveryTag, + redelivered = false}, _} -> + wait_for_messages(Config, [[QName, <<"1">>, <<"0">>, <<"1">>]]), + amqp_channel:cast(Ch, #'basic.nack'{delivery_tag = DeliveryTag, + multiple = false, + requeue = true}), + receive + {#'basic.deliver'{delivery_tag = DeliveryTag1, + redelivered = true}, _} -> + wait_for_messages(Config, [[QName, <<"1">>, <<"0">>, <<"1">>]]), + amqp_channel:cast(Ch, #'basic.ack'{delivery_tag = DeliveryTag1}), + wait_for_messages(Config, [[QName, <<"0">>, <<"0">>, <<"0">>]]) + end + end, + rabbit_ct_client_helpers:close_channel(Ch), + ok. + +subscribe_and_nack(Config) -> + {_, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + QName = ?config(queue_name, Config), + CArgs = ?config(consumer_args, Config), + declare_queue(Ch, Config, QName), + + publish(Ch, QName, [<<"msg1">>]), + wait_for_messages(Config, [[QName, <<"1">>, <<"1">>, <<"0">>]]), + subscribe(Ch, QName, false, CArgs), + receive + {#'basic.deliver'{delivery_tag = DeliveryTag, + redelivered = false}, _} -> + wait_for_messages(Config, [[QName, <<"1">>, <<"0">>, <<"1">>]]), + amqp_channel:cast(Ch, #'basic.nack'{delivery_tag = DeliveryTag, + multiple = false, + requeue = false}), + wait_for_messages(Config, [[QName, <<"0">>, <<"0">>, <<"0">>]]) + end, + rabbit_ct_client_helpers:close_channel(Ch), + ok. + +subscribe_and_multiple_nack(Config) -> + {_, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + QName = ?config(queue_name, Config), + CArgs = ?config(consumer_args, Config), + declare_queue(Ch, Config, QName), + + publish(Ch, QName, [<<"msg1">>, <<"msg2">>, <<"msg3">>]), + wait_for_messages(Config, [[QName, <<"3">>, <<"3">>, <<"0">>]]), + subscribe(Ch, QName, false, CArgs), + receive_basic_deliver(false), + receive_basic_deliver(false), + receive + {#'basic.deliver'{delivery_tag = DeliveryTag, + redelivered = false}, _} -> + wait_for_messages(Config, [[QName, <<"3">>, <<"0">>, <<"3">>]]), + amqp_channel:cast(Ch, #'basic.nack'{delivery_tag = DeliveryTag, + multiple = true, + requeue = false}), + wait_for_messages(Config, [[QName, <<"0">>, <<"0">>, <<"0">>]]) + end, + rabbit_ct_client_helpers:close_channel(Ch), + ok. + +%% TODO test with single active +basic_cancel(Config) -> + [Server | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + {_, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + QName = ?config(queue_name, Config), + CArgs = ?config(consumer_args, Config), + declare_queue(Ch, Config, QName), + + publish(Ch, QName, [<<"msg1">>]), + wait_for_messages(Config, [[QName, <<"1">>, <<"1">>, <<"0">>]]), + CTag = atom_to_binary(?FUNCTION_NAME, utf8), + + subscribe(Ch, QName, false, CTag, CArgs), + receive + {#'basic.deliver'{delivery_tag = DeliveryTag}, _} -> + wait_for_messages(Config, [[QName, <<"1">>, <<"0">>, <<"1">>]]), + amqp_channel:call(Ch, #'basic.cancel'{consumer_tag = CTag}), + Consumers = rpc:call(Server, rabbit_amqqueue, consumers_all, [<<"/">>]), + wait_for_messages(Config, [[QName, <<"1">>, <<"0">>, <<"1">>]]), + ?assertEqual([], lists:filter(fun(Props) -> + Resource = proplists:get_value(queue_name, Props), + QName == Resource#resource.name + end, Consumers)), + publish(Ch, QName, [<<"msg2">>, <<"msg3">>]), + wait_for_messages(Config, [[QName, <<"3">>, <<"2">>, <<"1">>]]), + amqp_channel:cast(Ch, #'basic.ack'{delivery_tag = DeliveryTag}), + wait_for_messages(Config, [[QName, <<"2">>, <<"2">>, <<"0">>]]) + after 5000 -> + exit(basic_deliver_timeout) + end, + rabbit_ct_client_helpers:close_channel(Ch), + ok. + +purge(Config) -> + {_, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + QName = ?config(queue_name, Config), + declare_queue(Ch, Config, QName), + + publish(Ch, QName, [<<"msg1">>, <<"msg2">>]), + wait_for_messages(Config, [[QName, <<"2">>, <<"2">>, <<"0">>]]), + [_] = consume(Ch, QName, [<<"msg1">>]), + wait_for_messages(Config, [[QName, <<"2">>, <<"1">>, <<"1">>]]), + {'queue.purge_ok', 1} = amqp_channel:call(Ch, #'queue.purge'{queue = QName}), + wait_for_messages(Config, [[QName, <<"1">>, <<"0">>, <<"1">>]]), + rabbit_ct_client_helpers:close_channel(Ch), + ok. + +basic_recover(Config) -> + {_, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + QName = ?config(queue_name, Config), + declare_queue(Ch, Config, QName), + + publish(Ch, QName, [<<"msg1">>]), + wait_for_messages(Config, [[QName, <<"1">>, <<"1">>, <<"0">>]]), + [_] = consume(Ch, QName, [<<"msg1">>]), + wait_for_messages(Config, [[QName, <<"1">>, <<"0">>, <<"1">>]]), + amqp_channel:cast(Ch, #'basic.recover'{requeue = true}), + wait_for_messages(Config, [[QName, <<"1">>, <<"1">>, <<"0">>]]), + rabbit_ct_client_helpers:close_channel(Ch), + ok. + +delete_immediately_by_pid_fails(Config) -> + {_, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + Args = ?config(queue_args, Config), + Durable = ?config(queue_durable, Config), + QName = ?config(queue_name, Config), + declare_queue(Ch, Config, QName), + + Cmd = ["eval", "{ok, Q} = rabbit_amqqueue:lookup(rabbit_misc:r(<<\"/\">>, queue, <<\"" ++ binary_to_list(QName) ++ "\">>)), Pid = rabbit_amqqueue:pid_of(Q), rabbit_amqqueue:delete_immediately([Pid])."], + {ok, Msg} = rabbit_ct_broker_helpers:rabbitmqctl(Config, 0, Cmd), + ?assertEqual(match, re:run(Msg, ".*error.*", [{capture, none}])), + + ?assertEqual({'queue.declare_ok', QName, 0, 0}, + amqp_channel:call(Ch, #'queue.declare'{queue = QName, + durable = Durable, + passive = true, + auto_delete = false, + arguments = Args})), + rabbit_ct_client_helpers:close_channel(Ch), + ok. + +delete_immediately_by_pid_succeeds(Config) -> + {_, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + Args = ?config(queue_args, Config), + Durable = ?config(queue_durable, Config), + QName = ?config(queue_name, Config), + declare_queue(Ch, Config, QName), + + Cmd = ["eval", "{ok, Q} = rabbit_amqqueue:lookup(rabbit_misc:r(<<\"/\">>, queue, <<\"" ++ binary_to_list(QName) ++ "\">>)), Pid = rabbit_amqqueue:pid_of(Q), rabbit_amqqueue:delete_immediately([Pid])."], + {ok, Msg} = rabbit_ct_broker_helpers:rabbitmqctl(Config, 0, Cmd), + ?assertEqual(match, re:run(Msg, ".*ok.*", [{capture, none}])), + + ?assertExit( + {{shutdown, {server_initiated_close, 404, _}}, _}, + amqp_channel:call(Ch, #'queue.declare'{queue = QName, + durable = Durable, + passive = true, + auto_delete = false, + arguments = Args})), + rabbit_ct_client_helpers:close_channel(Ch), + ok. + +delete_immediately_by_resource(Config) -> + {_, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), + Args = ?config(queue_args, Config), + Durable = ?config(queue_durable, Config), + QName = ?config(queue_name, Config), + declare_queue(Ch, Config, QName), + + Cmd = ["eval", "rabbit_amqqueue:delete_immediately_by_resource([rabbit_misc:r(<<\"/\">>, queue, <<\"" ++ binary_to_list(QName) ++ "\">>)])."], + ?assertEqual({ok, "ok\n"}, rabbit_ct_broker_helpers:rabbitmqctl(Config, 0, Cmd)), + + ?assertExit( + {{shutdown, {server_initiated_close, 404, _}}, _}, + amqp_channel:call(Ch, #'queue.declare'{queue = QName, + durable = Durable, + passive = true, + auto_delete = false, + arguments = Args})), + rabbit_ct_client_helpers:close_channel(Ch), + ok. + +%%%%%%%%%%%%%%%%%%%%%%%% +%% Test helpers +%%%%%%%%%%%%%%%%%%%%%%%% +declare_queue(Ch, Config, QName) -> + Args = ?config(queue_args, Config), + Durable = ?config(queue_durable, Config), + #'queue.declare_ok'{} = amqp_channel:call(Ch, #'queue.declare'{queue = QName, + arguments = Args, + durable = Durable}). + +publish(Ch, QName, Payloads) -> + [amqp_channel:call(Ch, #'basic.publish'{routing_key = QName}, #amqp_msg{payload = Payload}) + || Payload <- Payloads]. + +consume(Ch, QName, Payloads) -> + consume(Ch, QName, false, Payloads). + +consume(Ch, QName, NoAck, Payloads) -> + [begin + {#'basic.get_ok'{delivery_tag = DTag}, #amqp_msg{payload = Payload}} = + amqp_channel:call(Ch, #'basic.get'{queue = QName, + no_ack = NoAck}), + DTag + end || Payload <- Payloads]. + +consume_empty(Ch, QName) -> + ?assertMatch(#'basic.get_empty'{}, + amqp_channel:call(Ch, #'basic.get'{queue = QName})). + +subscribe(Ch, Queue, NoAck, CArgs) -> + subscribe(Ch, Queue, NoAck, <<"ctag">>, CArgs). + +subscribe(Ch, Queue, NoAck, Ctag, CArgs) -> + amqp_channel:subscribe(Ch, #'basic.consume'{queue = Queue, + no_ack = NoAck, + consumer_tag = Ctag, + arguments = CArgs}, + self()), + receive + #'basic.consume_ok'{consumer_tag = Ctag} -> + ok + end. + +receive_basic_deliver(Redelivered) -> + receive + {#'basic.deliver'{redelivered = R}, _} when R == Redelivered -> + ok + end. + +flush(T) -> + receive X -> + ct:pal("flushed ~w", [X]), + flush(T) + after T -> + ok + end. diff --git a/deps/rabbit/test/queue_type_SUITE.erl b/deps/rabbit/test/queue_type_SUITE.erl new file mode 100644 index 0000000000..aed5ad4ccb --- /dev/null +++ b/deps/rabbit/test/queue_type_SUITE.erl @@ -0,0 +1,275 @@ +-module(queue_type_SUITE). + +-compile(export_all). + +-export([ + ]). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). + +%%%=================================================================== +%%% Common Test callbacks +%%%=================================================================== + +all() -> + [ + {group, classic}, + {group, quorum} + ]. + + +all_tests() -> + [ + smoke, + ack_after_queue_delete + ]. + +groups() -> + [ + {classic, [], all_tests()}, + {quorum, [], all_tests()} + ]. + +init_per_suite(Config0) -> + rabbit_ct_helpers:log_environment(), + Config = rabbit_ct_helpers:merge_app_env( + Config0, {rabbit, [{quorum_tick_interval, 1000}]}), + rabbit_ct_helpers:run_setup_steps(Config). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config), + ok. + +init_per_group(Group, Config) -> + ClusterSize = 3, + Config1 = rabbit_ct_helpers:set_config(Config, + [{rmq_nodes_count, ClusterSize}, + {rmq_nodename_suffix, Group}, + {tcp_ports_base}]), + Config1b = rabbit_ct_helpers:set_config(Config1, + [{queue_type, atom_to_binary(Group, utf8)}, + {net_ticktime, 10}]), + Config2 = rabbit_ct_helpers:run_steps(Config1b, + [fun merge_app_env/1 ] ++ + rabbit_ct_broker_helpers:setup_steps()), + case rabbit_ct_broker_helpers:enable_feature_flag(Config2, quorum_queue) of + ok -> + ok = rabbit_ct_broker_helpers:rpc( + Config2, 0, application, set_env, + [rabbit, channel_tick_interval, 100]), + %% HACK: the larger cluster sizes benefit for a bit more time + %% after clustering before running the tests. + Config3 = case Group of + cluster_size_5 -> + timer:sleep(5000), + Config2; + _ -> + Config2 + end, + + rabbit_ct_broker_helpers:set_policy( + Config3, 0, + <<"ha-policy">>, <<".*">>, <<"queues">>, + [{<<"ha-mode">>, <<"all">>}]), + Config3; + Skip -> + end_per_group(Group, Config2), + Skip + end. + +merge_app_env(Config) -> + rabbit_ct_helpers:merge_app_env( + rabbit_ct_helpers:merge_app_env(Config, + {rabbit, + [{core_metrics_gc_interval, 100}, + {log, [{file, [{level, debug}]}]}]}), + {ra, [{min_wal_roll_over_interval, 30000}]}). + +end_per_group(_Group, Config) -> + rabbit_ct_helpers:run_steps(Config, + rabbit_ct_broker_helpers:teardown_steps()). + +init_per_testcase(Testcase, Config) -> + Config1 = rabbit_ct_helpers:testcase_started(Config, Testcase), + rabbit_ct_broker_helpers:rpc(Config, 0, ?MODULE, delete_queues, []), + Q = rabbit_data_coercion:to_binary(Testcase), + Config2 = rabbit_ct_helpers:set_config(Config1, + [{queue_name, Q}, + {alt_queue_name, <<Q/binary, "_alt">>} + ]), + rabbit_ct_helpers:run_steps(Config2, + rabbit_ct_client_helpers:setup_steps()). + +end_per_testcase(Testcase, Config) -> + catch delete_queues(), + Config1 = rabbit_ct_helpers:run_steps( + Config, + rabbit_ct_client_helpers:teardown_steps()), + rabbit_ct_helpers:testcase_finished(Config1, Testcase). + +%%%=================================================================== +%%% Test cases +%%%=================================================================== + +smoke(Config) -> + Server = rabbit_ct_broker_helpers:get_node_config(Config, 0, nodename), + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + QName = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', QName, 0, 0}, + declare(Ch, QName, [{<<"x-queue-type">>, longstr, + ?config(queue_type, Config)}])), + #'confirm.select_ok'{} = amqp_channel:call(Ch, #'confirm.select'{}), + amqp_channel:register_confirm_handler(Ch, self()), + publish(Ch, QName, <<"msg1">>), + ct:pal("waiting for confirms from ~s", [QName]), + ok = receive + #'basic.ack'{} -> ok; + #'basic.nack'{} -> fail + after 2500 -> + flush(), + exit(confirm_timeout) + end, + DTag = basic_get(Ch, QName), + + basic_ack(Ch, DTag), + basic_get_empty(Ch, QName), + + %% consume + publish(Ch, QName, <<"msg2">>), + ConsumerTag1 = <<"ctag1">>, + ok = subscribe(Ch, QName, ConsumerTag1), + %% receive and ack + receive + {#'basic.deliver'{delivery_tag = DeliveryTag, + redelivered = false}, + #amqp_msg{}} -> + basic_ack(Ch, DeliveryTag) + after 5000 -> + flush(), + exit(basic_deliver_timeout) + end, + basic_cancel(Ch, ConsumerTag1), + + %% assert empty + basic_get_empty(Ch, QName), + + %% consume and nack + ConsumerTag2 = <<"ctag2">>, + ok = subscribe(Ch, QName, ConsumerTag2), + publish(Ch, QName, <<"msg3">>), + receive + {#'basic.deliver'{delivery_tag = T, + redelivered = false}, + #amqp_msg{}} -> + basic_cancel(Ch, ConsumerTag2), + basic_nack(Ch, T) + after 5000 -> + exit(basic_deliver_timeout) + end, + %% get and ack + basic_ack(Ch, basic_get(Ch, QName)), + ok. + +ack_after_queue_delete(Config) -> + Server = rabbit_ct_broker_helpers:get_node_config(Config, 0, nodename), + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + QName = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', QName, 0, 0}, + declare(Ch, QName, [{<<"x-queue-type">>, longstr, + ?config(queue_type, Config)}])), + #'confirm.select_ok'{} = amqp_channel:call(Ch, #'confirm.select'{}), + amqp_channel:register_confirm_handler(Ch, self()), + publish(Ch, QName, <<"msg1">>), + ct:pal("waiting for confirms from ~s", [QName]), + ok = receive + #'basic.ack'{} -> ok; + #'basic.nack'{} -> + ct:fail("confirm nack - expected ack") + after 2500 -> + flush(), + exit(confirm_timeout) + end, + + DTag = basic_get(Ch, QName), + + ChRef = erlang:monitor(process, Ch), + #'queue.delete_ok'{} = delete(Ch, QName), + + basic_ack(Ch, DTag), + %% assert no channel error + receive + {'DOWN', ChRef, process, _, _} -> + ct:fail("unexpected channel closure") + after 1000 -> + ok + end, + flush(), + ok. + +%% Utility +%% +delete_queues() -> + [rabbit_amqqueue:delete(Q, false, false, <<"dummy">>) + || Q <- rabbit_amqqueue:list()]. + +declare(Ch, Q, Args) -> + amqp_channel:call(Ch, #'queue.declare'{queue = Q, + durable = true, + auto_delete = false, + arguments = Args}). + +delete(Ch, Q) -> + amqp_channel:call(Ch, #'queue.delete'{queue = Q}). + +publish(Ch, Queue, Msg) -> + ok = amqp_channel:cast(Ch, + #'basic.publish'{routing_key = Queue}, + #amqp_msg{props = #'P_basic'{delivery_mode = 2}, + payload = Msg}). + +basic_get(Ch, Queue) -> + {GetOk, _} = Reply = amqp_channel:call(Ch, #'basic.get'{queue = Queue, + no_ack = false}), + ?assertMatch({#'basic.get_ok'{}, #amqp_msg{}}, Reply), + GetOk#'basic.get_ok'.delivery_tag. + +basic_get_empty(Ch, Queue) -> + ?assertMatch(#'basic.get_empty'{}, + amqp_channel:call(Ch, #'basic.get'{queue = Queue, + no_ack = false})). + +subscribe(Ch, Queue, CTag) -> + amqp_channel:subscribe(Ch, #'basic.consume'{queue = Queue, + no_ack = false, + consumer_tag = CTag}, + self()), + receive + #'basic.consume_ok'{consumer_tag = CTag} -> + ok + after 5000 -> + exit(basic_consume_timeout) + end. + +basic_ack(Ch, DTag) -> + amqp_channel:cast(Ch, #'basic.ack'{delivery_tag = DTag, + multiple = false}). + +basic_cancel(Ch, CTag) -> + #'basic.cancel_ok'{} = + amqp_channel:call(Ch, #'basic.cancel'{consumer_tag = CTag}). + +basic_nack(Ch, DTag) -> + amqp_channel:cast(Ch, #'basic.nack'{delivery_tag = DTag, + requeue = true, + multiple = false}). + +flush() -> + receive + Any -> + ct:pal("flush ~p", [Any]), + flush() + after 0 -> + ok + end. diff --git a/deps/rabbit/test/quorum_queue_SUITE.erl b/deps/rabbit/test/quorum_queue_SUITE.erl new file mode 100644 index 0000000000..36a6d41a61 --- /dev/null +++ b/deps/rabbit/test/quorum_queue_SUITE.erl @@ -0,0 +1,2792 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2018-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(quorum_queue_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). + +-import(quorum_queue_utils, [wait_for_messages_ready/3, + wait_for_messages_pending_ack/3, + wait_for_messages_total/3, + wait_for_messages/2, + dirty_query/3, + ra_name/1, + is_mixed_versions/0]). + +-compile(export_all). + +suite() -> + [{timetrap, 5 * 60000}]. + +all() -> + [ + {group, single_node}, + {group, unclustered}, + {group, clustered} + ]. + +groups() -> + [ + {single_node, [], all_tests()}, + {single_node, [], memory_tests()}, + {single_node, [], [node_removal_is_quorum_critical]}, + {unclustered, [], [ + {cluster_size_2, [], [add_member]} + ]}, + {clustered, [], [ + {cluster_size_2, [], [cleanup_data_dir]}, + {cluster_size_2, [], [add_member_not_running, + add_member_classic, + add_member_already_a_member, + add_member_not_found, + delete_member_not_running, + delete_member_classic, + delete_member_queue_not_found, + delete_member, + delete_member_not_a_member, + node_removal_is_quorum_critical] + ++ all_tests()}, + {cluster_size_2, [], memory_tests()}, + {cluster_size_3, [], [ + declare_during_node_down, + simple_confirm_availability_on_leader_change, + publishing_to_unavailable_queue, + confirm_availability_on_leader_change, + recover_from_single_failure, + recover_from_multiple_failures, + leadership_takeover, + delete_declare, + delete_member_during_node_down, + metrics_cleanup_on_leadership_takeover, + metrics_cleanup_on_leader_crash, + consume_in_minority, + shrink_all, + rebalance, + file_handle_reservations, + file_handle_reservations_above_limit, + node_removal_is_not_quorum_critical + ]}, + {cluster_size_5, [], [start_queue, + start_queue_concurrent, + quorum_cluster_size_3, + quorum_cluster_size_7, + node_removal_is_not_quorum_critical + ]}, + {clustered_with_partitions, [], [ + reconnect_consumer_and_publish, + reconnect_consumer_and_wait, + reconnect_consumer_and_wait_channel_down + ]} + ]} + ]. + +all_tests() -> + [ + declare_args, + declare_invalid_properties, + declare_server_named, + start_queue, + stop_queue, + restart_queue, + restart_all_types, + stop_start_rabbit_app, + publish_and_restart, + subscribe_should_fail_when_global_qos_true, + dead_letter_to_classic_queue, + dead_letter_with_memory_limit, + dead_letter_to_quorum_queue, + dead_letter_from_classic_to_quorum_queue, + dead_letter_policy, + cleanup_queue_state_on_channel_after_publish, + cleanup_queue_state_on_channel_after_subscribe, + sync_queue, + cancel_sync_queue, + idempotent_recover, + vhost_with_quorum_queue_is_deleted, + delete_immediately_by_resource, + consume_redelivery_count, + subscribe_redelivery_count, + message_bytes_metrics, + queue_length_limit_drop_head, + queue_length_limit_reject_publish, + subscribe_redelivery_limit, + subscribe_redelivery_policy, + subscribe_redelivery_limit_with_dead_letter, + queue_length_in_memory_limit_basic_get, + queue_length_in_memory_limit_subscribe, + queue_length_in_memory_limit, + queue_length_in_memory_limit_returns, + queue_length_in_memory_bytes_limit_basic_get, + queue_length_in_memory_bytes_limit_subscribe, + queue_length_in_memory_bytes_limit, + queue_length_in_memory_purge, + in_memory, + consumer_metrics, + invalid_policy, + delete_if_empty, + delete_if_unused, + queue_ttl, + peek, + consumer_priorities + ]. + +memory_tests() -> + [ + memory_alarm_rolls_wal + ]. + +%% ------------------------------------------------------------------- +%% Testsuite setup/teardown. +%% ------------------------------------------------------------------- + +init_per_suite(Config0) -> + rabbit_ct_helpers:log_environment(), + Config1 = rabbit_ct_helpers:merge_app_env( + Config0, {rabbit, [{quorum_tick_interval, 1000}]}), + Config = rabbit_ct_helpers:merge_app_env( + Config1, {aten, [{poll_interval, 1000}]}), + rabbit_ct_helpers:run_setup_steps( + Config, + [fun rabbit_ct_broker_helpers:configure_dist_proxy/1]). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(clustered, Config) -> + rabbit_ct_helpers:set_config(Config, [{rmq_nodes_clustered, true}]); +init_per_group(unclustered, Config) -> + rabbit_ct_helpers:set_config(Config, [{rmq_nodes_clustered, false}]); +init_per_group(clustered_with_partitions, Config) -> + case is_mixed_versions() of + true -> + {skip, "clustered_with_partitions is too unreliable in mixed mode"}; + false -> + rabbit_ct_helpers:set_config(Config, [{net_ticktime, 10}]) + end; +init_per_group(Group, Config) -> + ClusterSize = case Group of + single_node -> 1; + cluster_size_2 -> 2; + cluster_size_3 -> 3; + cluster_size_5 -> 5 + end, + IsMixed = not (false == os:getenv("SECONDARY_UMBRELLA")), + case ClusterSize of + 2 when IsMixed -> + {skip, "cluster size 2 isn't mixed versions compatible"}; + _ -> + Config1 = rabbit_ct_helpers:set_config(Config, + [{rmq_nodes_count, ClusterSize}, + {rmq_nodename_suffix, Group}, + {tcp_ports_base}]), + Config1b = rabbit_ct_helpers:set_config(Config1, [{net_ticktime, 10}]), + Ret = rabbit_ct_helpers:run_steps(Config1b, + [fun merge_app_env/1 ] ++ + rabbit_ct_broker_helpers:setup_steps()), + case Ret of + {skip, _} -> + Ret; + Config2 -> + EnableFF = rabbit_ct_broker_helpers:enable_feature_flag( + Config2, quorum_queue), + case EnableFF of + ok -> + ok = rabbit_ct_broker_helpers:rpc( + Config2, 0, application, set_env, + [rabbit, channel_tick_interval, 100]), + %% HACK: the larger cluster sizes benefit for a bit + %% more time after clustering before running the + %% tests. + case Group of + cluster_size_5 -> + timer:sleep(5000), + Config2; + _ -> + Config2 + end; + Skip -> + end_per_group(Group, Config2), + Skip + end + end + end. + +end_per_group(clustered, Config) -> + Config; +end_per_group(unclustered, Config) -> + Config; +end_per_group(clustered_with_partitions, Config) -> + Config; +end_per_group(_, Config) -> + rabbit_ct_helpers:run_steps(Config, + rabbit_ct_broker_helpers:teardown_steps()). + +init_per_testcase(Testcase, Config) when Testcase == reconnect_consumer_and_publish; + Testcase == reconnect_consumer_and_wait; + Testcase == reconnect_consumer_and_wait_channel_down -> + Config1 = rabbit_ct_helpers:testcase_started(Config, Testcase), + Q = rabbit_data_coercion:to_binary(Testcase), + Config2 = rabbit_ct_helpers:set_config(Config1, + [{rmq_nodes_count, 3}, + {rmq_nodename_suffix, Testcase}, + {tcp_ports_base}, + {queue_name, Q}, + {alt_queue_name, <<Q/binary, "_alt">>} + ]), + Ret = rabbit_ct_helpers:run_steps( + Config2, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()), + case Ret of + {skip, _} -> + Ret; + Config3 -> + EnableFF = rabbit_ct_broker_helpers:enable_feature_flag( + Config3, quorum_queue), + case EnableFF of + ok -> + Config3; + Skip -> + end_per_testcase(Testcase, Config3), + Skip + end + end; +init_per_testcase(Testcase, Config) -> + Config1 = rabbit_ct_helpers:testcase_started(Config, Testcase), + rabbit_ct_broker_helpers:rpc(Config, 0, ?MODULE, delete_queues, []), + Q = rabbit_data_coercion:to_binary(Testcase), + Config2 = rabbit_ct_helpers:set_config(Config1, + [{queue_name, Q}, + {alt_queue_name, <<Q/binary, "_alt">>} + ]), + rabbit_ct_helpers:run_steps(Config2, rabbit_ct_client_helpers:setup_steps()). + +merge_app_env(Config) -> + rabbit_ct_helpers:merge_app_env( + rabbit_ct_helpers:merge_app_env(Config, + {rabbit, [{core_metrics_gc_interval, 100}]}), + {ra, [{min_wal_roll_over_interval, 30000}]}). + +end_per_testcase(Testcase, Config) when Testcase == reconnect_consumer_and_publish; + Testcase == reconnect_consumer_and_wait; + Testcase == reconnect_consumer_and_wait_channel_down -> + Config1 = rabbit_ct_helpers:run_steps(Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()), + rabbit_ct_helpers:testcase_finished(Config1, Testcase); +end_per_testcase(Testcase, Config) -> + catch delete_queues(), + Config1 = rabbit_ct_helpers:run_steps( + Config, + rabbit_ct_client_helpers:teardown_steps()), + rabbit_ct_helpers:testcase_finished(Config1, Testcase). + +%% ------------------------------------------------------------------- +%% Testcases. +%% ------------------------------------------------------------------- + +declare_args(Config) -> + Server = rabbit_ct_broker_helpers:get_node_config(Config, 0, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + LQ = ?config(queue_name, Config), + declare(Ch, LQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}, + {<<"x-max-length">>, long, 2000}, + {<<"x-max-length-bytes">>, long, 2000}]), + assert_queue_type(Server, LQ, rabbit_quorum_queue), + + DQ = <<"classic-declare-args-q">>, + declare(Ch, DQ, [{<<"x-queue-type">>, longstr, <<"classic">>}]), + assert_queue_type(Server, DQ, rabbit_classic_queue), + + DQ2 = <<"classic-q2">>, + declare(Ch, DQ2), + assert_queue_type(Server, DQ2, rabbit_classic_queue). + +declare_invalid_properties(Config) -> + Server = rabbit_ct_broker_helpers:get_node_config(Config, 0, nodename), + LQ = ?config(queue_name, Config), + + ?assertExit( + {{shutdown, {server_initiated_close, 406, _}}, _}, + amqp_channel:call( + rabbit_ct_client_helpers:open_channel(Config, Server), + #'queue.declare'{queue = LQ, + auto_delete = true, + durable = true, + arguments = [{<<"x-queue-type">>, longstr, <<"quorum">>}]})), + ?assertExit( + {{shutdown, {server_initiated_close, 406, _}}, _}, + amqp_channel:call( + rabbit_ct_client_helpers:open_channel(Config, Server), + #'queue.declare'{queue = LQ, + exclusive = true, + durable = true, + arguments = [{<<"x-queue-type">>, longstr, <<"quorum">>}]})), + ?assertExit( + {{shutdown, {server_initiated_close, 406, _}}, _}, + amqp_channel:call( + rabbit_ct_client_helpers:open_channel(Config, Server), + #'queue.declare'{queue = LQ, + durable = false, + arguments = [{<<"x-queue-type">>, longstr, <<"quorum">>}]})). + +declare_server_named(Config) -> + Server = rabbit_ct_broker_helpers:get_node_config(Config, 0, nodename), + + ?assertExit( + {{shutdown, {server_initiated_close, 406, _}}, _}, + declare(rabbit_ct_client_helpers:open_channel(Config, Server), + <<"">>, [{<<"x-queue-type">>, longstr, <<"quorum">>}])). + +start_queue(Config) -> + Server = rabbit_ct_broker_helpers:get_node_config(Config, 0, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + LQ = ?config(queue_name, Config), + %% The stream coordinator is also a ra process, we need to ensure the quorum tests + %% are not affected by any other ra cluster that could be added in the future + Children = length(rpc:call(Server, supervisor, which_children, [ra_server_sup_sup])), + + ?assertEqual({'queue.declare_ok', LQ, 0, 0}, + declare(Ch, LQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), + + %% Check that the application and one ra node are up + ?assertMatch({ra, _, _}, lists:keyfind(ra, 1, + rpc:call(Server, application, which_applications, []))), + Expected = Children + 1, + ?assertMatch(Expected, + length(rpc:call(Server, supervisor, which_children, [ra_server_sup_sup]))), + + %% Test declare an existing queue + ?assertEqual({'queue.declare_ok', LQ, 0, 0}, + declare(Ch, LQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), + + %% Test declare with same arguments + ?assertEqual({'queue.declare_ok', LQ, 0, 0}, + declare(Ch, LQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), + + %% Test declare an existing queue with different arguments + ?assertExit(_, declare(Ch, LQ, [])), + + %% Check that the application and process are still up + ?assertMatch({ra, _, _}, lists:keyfind(ra, 1, + rpc:call(Server, application, which_applications, []))), + ?assertMatch(Expected, + length(rpc:call(Server, supervisor, which_children, [ra_server_sup_sup]))). + +start_queue_concurrent(Config) -> + Servers = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + LQ = ?config(queue_name, Config), + Self = self(), + [begin + _ = spawn_link(fun () -> + {_Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, Server), + %% Test declare an existing queue + ?assertEqual({'queue.declare_ok', LQ, 0, 0}, + declare(Ch, LQ, + [{<<"x-queue-type">>, + longstr, + <<"quorum">>}])), + Self ! {done, Server} + end) + end || Server <- Servers], + + [begin + receive {done, Server} -> ok + after 5000 -> exit({await_done_timeout, Server}) + end + end || Server <- Servers], + + + ok. + +quorum_cluster_size_3(Config) -> + case is_mixed_versions() of + true -> + {skip, "quorum_cluster_size_3 tests isn't mixed version reliable"}; + false -> + quorum_cluster_size_x(Config, 3, 3) + end. + +quorum_cluster_size_7(Config) -> + quorum_cluster_size_x(Config, 7, 5). + +quorum_cluster_size_x(Config, Max, Expected) -> + Server = rabbit_ct_broker_helpers:get_node_config(Config, 0, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + QQ = ?config(queue_name, Config), + RaName = ra_name(QQ), + ?assertEqual({'queue.declare_ok', QQ, 0, 0}, + declare(Ch, QQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}, + {<<"x-quorum-initial-group-size">>, long, Max}])), + {ok, Members, _} = ra:members({RaName, Server}), + ?assertEqual(Expected, length(Members)), + Info = rpc:call(Server, rabbit_quorum_queue, infos, + [rabbit_misc:r(<<"/">>, queue, QQ)]), + MembersQ = proplists:get_value(members, Info), + ?assertEqual(Expected, length(MembersQ)). + +stop_queue(Config) -> + Server = rabbit_ct_broker_helpers:get_node_config(Config, 0, nodename), + + %% The stream coordinator is also a ra process, we need to ensure the quorum tests + %% are not affected by any other ra cluster that could be added in the future + Children = length(rpc:call(Server, supervisor, which_children, [ra_server_sup_sup])), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + LQ = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', LQ, 0, 0}, + declare(Ch, LQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), + + %% Check that the application and one ra node are up + ?assertMatch({ra, _, _}, lists:keyfind(ra, 1, + rpc:call(Server, application, which_applications, []))), + Expected = Children + 1, + ?assertMatch(Expected, + length(rpc:call(Server, supervisor, which_children, [ra_server_sup_sup]))), + + %% Delete the quorum queue + ?assertMatch(#'queue.delete_ok'{}, amqp_channel:call(Ch, #'queue.delete'{queue = LQ})), + %% Check that the application and process are down + wait_until(fun() -> + Children == length(rpc:call(Server, supervisor, which_children, [ra_server_sup_sup])) + end), + ?assertMatch({ra, _, _}, lists:keyfind(ra, 1, + rpc:call(Server, application, which_applications, []))). + +restart_queue(Config) -> + Server = rabbit_ct_broker_helpers:get_node_config(Config, 0, nodename), + + %% The stream coordinator is also a ra process, we need to ensure the quorum tests + %% are not affected by any other ra cluster that could be added in the future + Children = length(rpc:call(Server, supervisor, which_children, [ra_server_sup_sup])), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + LQ = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', LQ, 0, 0}, + declare(Ch, LQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), + + ok = rabbit_ct_broker_helpers:stop_node(Config, Server), + ok = rabbit_ct_broker_helpers:start_node(Config, Server), + + %% Check that the application and one ra node are up + ?assertMatch({ra, _, _}, lists:keyfind(ra, 1, + rpc:call(Server, application, which_applications, []))), + Expected = Children + 1, + ?assertMatch(Expected, + length(rpc:call(Server, supervisor, which_children, [ra_server_sup_sup]))), + Ch2 = rabbit_ct_client_helpers:open_channel(Config, Server), + delete_queues(Ch2, [LQ]). + +idempotent_recover(Config) -> + Server = rabbit_ct_broker_helpers:get_node_config(Config, 0, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + LQ = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', LQ, 0, 0}, + declare(Ch, LQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), + + %% kill default vhost to trigger recovery + [{_, SupWrapperPid, _, _} | _] = rpc:call(Server, supervisor, + which_children, + [rabbit_vhost_sup_sup]), + [{_, Pid, _, _} | _] = rpc:call(Server, supervisor, + which_children, + [SupWrapperPid]), + %% kill the vhost process to trigger recover + rpc:call(Server, erlang, exit, [Pid, kill]), + + timer:sleep(1000), + %% validate quorum queue is still functional + RaName = ra_name(LQ), + {ok, _, _} = ra:members({RaName, Server}), + %% validate vhosts are running - or rather validate that at least one + %% vhost per cluster is running + [begin + #{cluster_state := ServerStatuses} = maps:from_list(I), + ?assertMatch(#{Server := running}, maps:from_list(ServerStatuses)) + end || I <- rpc:call(Server, rabbit_vhost,info_all, [])], + ok. + +vhost_with_quorum_queue_is_deleted(Config) -> + Node = rabbit_ct_broker_helpers:get_node_config(Config, 0, nodename), + VHost = <<"vhost2">>, + QName = atom_to_binary(?FUNCTION_NAME, utf8), + RaName = binary_to_atom(<<VHost/binary, "_", QName/binary>>, utf8), + User = ?config(rmq_username, Config), + ok = rabbit_ct_broker_helpers:add_vhost(Config, Node, VHost, User), + ok = rabbit_ct_broker_helpers:set_full_permissions(Config, User, VHost), + Conn = rabbit_ct_client_helpers:open_unmanaged_connection(Config, Node, + VHost), + {ok, Ch} = amqp_connection:open_channel(Conn), + ?assertEqual({'queue.declare_ok', QName, 0, 0}, + declare(Ch, QName, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), + + UId = rpc:call(Node, ra_directory, where_is, [RaName]), + ?assert(UId =/= undefined), + ok = rabbit_ct_broker_helpers:delete_vhost(Config, VHost), + %% validate quorum queues got deleted + undefined = rpc:call(Node, ra_directory, where_is, [RaName]), + ok. + +restart_all_types(Config) -> + %% Test the node restart with both types of queues (quorum and classic) to + %% ensure there are no regressions + Server = rabbit_ct_broker_helpers:get_node_config(Config, 0, nodename), + + %% The stream coordinator is also a ra process, we need to ensure the quorum tests + %% are not affected by any other ra cluster that could be added in the future + Children = rpc:call(Server, supervisor, which_children, [ra_server_sup_sup]), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + QQ1 = <<"restart_all_types-qq1">>, + ?assertEqual({'queue.declare_ok', QQ1, 0, 0}, + declare(Ch, QQ1, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), + QQ2 = <<"restart_all_types-qq2">>, + ?assertEqual({'queue.declare_ok', QQ2, 0, 0}, + declare(Ch, QQ2, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), + + CQ1 = <<"restart_all_types-classic1">>, + ?assertEqual({'queue.declare_ok', CQ1, 0, 0}, declare(Ch, CQ1, [])), + rabbit_ct_client_helpers:publish(Ch, CQ1, 1), + CQ2 = <<"restart_all_types-classic2">>, + ?assertEqual({'queue.declare_ok', CQ2, 0, 0}, declare(Ch, CQ2, [])), + rabbit_ct_client_helpers:publish(Ch, CQ2, 1), + + ok = rabbit_ct_broker_helpers:stop_node(Config, Server), + ok = rabbit_ct_broker_helpers:start_node(Config, Server), + + %% Check that the application and two ra nodes are up. Queues are restored + %% after the broker is marked as "ready", that's why we need to wait for + %% the condition. + ?assertMatch({ra, _, _}, lists:keyfind(ra, 1, + rpc:call(Server, application, which_applications, []))), + Expected = length(Children) + 2, + ok = rabbit_ct_helpers:await_condition( + fun() -> + Expected =:= length( + rpc:call( + Server, + supervisor, + which_children, + [ra_server_sup_sup])) + end, 60000), + %% Check the classic queues restarted correctly + Ch2 = rabbit_ct_client_helpers:open_channel(Config, Server), + {#'basic.get_ok'{}, #amqp_msg{}} = + amqp_channel:call(Ch2, #'basic.get'{queue = CQ1, no_ack = false}), + {#'basic.get_ok'{}, #amqp_msg{}} = + amqp_channel:call(Ch2, #'basic.get'{queue = CQ2, no_ack = false}), + delete_queues(Ch2, [QQ1, QQ2, CQ1, CQ2]). + +delete_queues(Ch, Queues) -> + [amqp_channel:call(Ch, #'queue.delete'{queue = Q}) || Q <- Queues], + ok. + +stop_start_rabbit_app(Config) -> + %% Test start/stop of rabbit app with both types of queues (quorum and + %% classic) to ensure there are no regressions + Server = rabbit_ct_broker_helpers:get_node_config(Config, 0, nodename), + + %% The stream coordinator is also a ra process, we need to ensure the quorum tests + %% are not affected by any other ra cluster that could be added in the future + Children = length(rpc:call(Server, supervisor, which_children, [ra_server_sup_sup])), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + QQ1 = <<"stop_start_rabbit_app-qq">>, + ?assertEqual({'queue.declare_ok', QQ1, 0, 0}, + declare(Ch, QQ1, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), + QQ2 = <<"quorum-q2">>, + ?assertEqual({'queue.declare_ok', QQ2, 0, 0}, + declare(Ch, QQ2, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), + + CQ1 = <<"stop_start_rabbit_app-classic">>, + ?assertEqual({'queue.declare_ok', CQ1, 0, 0}, declare(Ch, CQ1, [])), + rabbit_ct_client_helpers:publish(Ch, CQ1, 1), + CQ2 = <<"stop_start_rabbit_app-classic2">>, + ?assertEqual({'queue.declare_ok', CQ2, 0, 0}, declare(Ch, CQ2, [])), + rabbit_ct_client_helpers:publish(Ch, CQ2, 1), + + rabbit_control_helper:command(stop_app, Server), + %% Check the ra application has stopped (thus its supervisor and queues) + ?assertMatch(false, lists:keyfind(ra, 1, + rpc:call(Server, application, which_applications, []))), + + rabbit_control_helper:command(start_app, Server), + + %% Check that the application and two ra nodes are up + ?assertMatch({ra, _, _}, lists:keyfind(ra, 1, + rpc:call(Server, application, which_applications, []))), + Expected = Children + 2, + ?assertMatch(Expected, + length(rpc:call(Server, supervisor, which_children, [ra_server_sup_sup]))), + %% Check the classic queues restarted correctly + Ch2 = rabbit_ct_client_helpers:open_channel(Config, Server), + {#'basic.get_ok'{}, #amqp_msg{}} = + amqp_channel:call(Ch2, #'basic.get'{queue = CQ1, no_ack = false}), + {#'basic.get_ok'{}, #amqp_msg{}} = + amqp_channel:call(Ch2, #'basic.get'{queue = CQ2, no_ack = false}), + delete_queues(Ch2, [QQ1, QQ2, CQ1, CQ2]). + +publish_confirm(Ch, QName) -> + publish(Ch, QName), + amqp_channel:register_confirm_handler(Ch, self()), + ct:pal("waiting for confirms from ~s", [QName]), + receive + #'basic.ack'{} -> + ct:pal("CONFIRMED! ~s", [QName]), + ok; + #'basic.nack'{} -> + ct:pal("NOT CONFIRMED! ~s", [QName]), + fail + after 2500 -> + exit(confirm_timeout) + end. + +publish_and_restart(Config) -> + %% Test the node restart with both types of queues (quorum and classic) to + %% ensure there are no regressions + [Server | _] = Servers = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + QQ = ?config(queue_name, Config), + RaName = ra_name(QQ), + ?assertEqual({'queue.declare_ok', QQ, 0, 0}, + declare(Ch, QQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), + publish(Ch, QQ), + wait_for_messages_ready(Servers, RaName, 1), + wait_for_messages_pending_ack(Servers, RaName, 0), + + ok = rabbit_ct_broker_helpers:stop_node(Config, Server), + ok = rabbit_ct_broker_helpers:start_node(Config, Server), + + wait_for_messages_ready(Servers, RaName, 1), + wait_for_messages_pending_ack(Servers, RaName, 0), + publish(rabbit_ct_client_helpers:open_channel(Config, Server), QQ), + wait_for_messages_ready(Servers, RaName, 2), + wait_for_messages_pending_ack(Servers, RaName, 0). + +consume_in_minority(Config) -> + [Server0, Server1, Server2] = + rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server0), + QQ = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', QQ, 0, 0}, + declare(Ch, QQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), + + ok = rabbit_ct_broker_helpers:stop_node(Config, Server1), + ok = rabbit_ct_broker_helpers:stop_node(Config, Server2), + + ?assertExit({{shutdown, {connection_closing, {server_initiated_close, 541, _}}}, _}, + amqp_channel:call(Ch, #'basic.get'{queue = QQ, + no_ack = false})), + ok = rabbit_ct_broker_helpers:start_node(Config, Server1), + ok = rabbit_ct_broker_helpers:start_node(Config, Server2), + ok. + +shrink_all(Config) -> + [Server0, Server1, Server2] = + rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server0), + QQ = ?config(queue_name, Config), + AQ = ?config(alt_queue_name, Config), + ?assertEqual({'queue.declare_ok', QQ, 0, 0}, + declare(Ch, QQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), + ?assertEqual({'queue.declare_ok', AQ, 0, 0}, + declare(Ch, AQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), + timer:sleep(500), + Result = rpc:call(Server0, rabbit_quorum_queue, shrink_all, [Server2]), + ?assertMatch([{_, {ok, 2}}, {_, {ok, 2}}], Result), + Result1 = rpc:call(Server0, rabbit_quorum_queue, shrink_all, [Server1]), + ?assertMatch([{_, {ok, 1}}, {_, {ok, 1}}], Result1), + Result2 = rpc:call(Server0, rabbit_quorum_queue, shrink_all, [Server0]), + ?assertMatch([{_, {error, 1, last_node}}, + {_, {error, 1, last_node}}], Result2), + ok. + +rebalance(Config) -> + case is_mixed_versions() of + true -> + {skip, "rebalance tests isn't mixed version compatible"}; + false -> + rebalance0(Config) + end. + +rebalance0(Config) -> + [Server0, _, _] = + rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server0), + + Q1 = <<"q1">>, + Q2 = <<"q2">>, + Q3 = <<"q3">>, + Q4 = <<"q4">>, + Q5 = <<"q5">>, + + ?assertEqual({'queue.declare_ok', Q1, 0, 0}, + declare(Ch, Q1, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), + ?assertEqual({'queue.declare_ok', Q2, 0, 0}, + declare(Ch, Q2, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), + timer:sleep(1000), + + {ok, _, {_, Leader1}} = ra:members({ra_name(Q1), Server0}), + {ok, _, {_, Leader2}} = ra:members({ra_name(Q2), Server0}), + rabbit_ct_client_helpers:publish(Ch, Q1, 3), + rabbit_ct_client_helpers:publish(Ch, Q2, 2), + + ?assertEqual({'queue.declare_ok', Q3, 0, 0}, + declare(Ch, Q3, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), + ?assertEqual({'queue.declare_ok', Q4, 0, 0}, + declare(Ch, Q4, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), + ?assertEqual({'queue.declare_ok', Q5, 0, 0}, + declare(Ch, Q5, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), + timer:sleep(500), + {ok, Summary} = rpc:call(Server0, rabbit_amqqueue, rebalance, [quorum, ".*", ".*"]), + + %% Q1 and Q2 should not have moved leader, as these are the queues with more + %% log entries and we allow up to two queues per node (3 nodes, 5 queues) + ?assertMatch({ok, _, {_, Leader1}}, ra:members({ra_name(Q1), Server0})), + ?assertMatch({ok, _, {_, Leader2}}, ra:members({ra_name(Q2), Server0})), + + %% Check that we have at most 2 queues per node + ?assert(lists:all(fun(NodeData) -> + lists:all(fun({_, V}) when is_integer(V) -> V =< 2; + (_) -> true end, + NodeData) + end, Summary)), + ok. + +subscribe_should_fail_when_global_qos_true(Config) -> + [Server | _] = Servers = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + QQ = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', QQ, 0, 0}, + declare(Ch, QQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), + + RaName = ra_name(QQ), + qos(Ch, 10, true), + publish(Ch, QQ), + wait_for_messages_ready(Servers, RaName, 1), + wait_for_messages_pending_ack(Servers, RaName, 0), + try subscribe(Ch, QQ, false) of + _ -> exit(subscribe_should_not_pass) + catch + _:_ = Err -> + ct:pal("subscribe_should_fail_when_global_qos_true caught an error: ~p", [Err]) + end, + ok. + +dead_letter_to_classic_queue(Config) -> + [Server | _] = Servers = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + QQ = ?config(queue_name, Config), + CQ = <<"classic-dead_letter_to_classic_queue">>, + ?assertEqual({'queue.declare_ok', QQ, 0, 0}, + declare(Ch, QQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}, + {<<"x-dead-letter-exchange">>, longstr, <<>>}, + {<<"x-dead-letter-routing-key">>, longstr, CQ} + ])), + ?assertEqual({'queue.declare_ok', CQ, 0, 0}, declare(Ch, CQ, [])), + test_dead_lettering(true, Config, Ch, Servers, ra_name(QQ), QQ, CQ). + +dead_letter_with_memory_limit(Config) -> + [Server | _] = Servers = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + QQ = ?config(queue_name, Config), + CQ = <<"classic-dead_letter_with_memory_limit">>, + ?assertEqual({'queue.declare_ok', QQ, 0, 0}, + declare(Ch, QQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}, + {<<"x-max-in-memory-length">>, long, 0}, + {<<"x-dead-letter-exchange">>, longstr, <<>>}, + {<<"x-dead-letter-routing-key">>, longstr, CQ} + ])), + ?assertEqual({'queue.declare_ok', CQ, 0, 0}, declare(Ch, CQ, [])), + test_dead_lettering(true, Config, Ch, Servers, ra_name(QQ), QQ, CQ). + +test_dead_lettering(PolicySet, Config, Ch, Servers, RaName, Source, Destination) -> + publish(Ch, Source), + wait_for_messages_ready(Servers, RaName, 1), + wait_for_messages_pending_ack(Servers, RaName, 0), + wait_for_messages(Config, [[Destination, <<"0">>, <<"0">>, <<"0">>]]), + DeliveryTag = consume(Ch, Source, false), + wait_for_messages_ready(Servers, RaName, 0), + wait_for_messages_pending_ack(Servers, RaName, 1), + wait_for_messages(Config, [[Destination, <<"0">>, <<"0">>, <<"0">>]]), + amqp_channel:cast(Ch, #'basic.nack'{delivery_tag = DeliveryTag, + multiple = false, + requeue = false}), + wait_for_messages_ready(Servers, RaName, 0), + wait_for_messages_pending_ack(Servers, RaName, 0), + case PolicySet of + true -> + wait_for_messages(Config, [[Destination, <<"1">>, <<"1">>, <<"0">>]]), + _ = consume(Ch, Destination, true); + false -> + wait_for_messages(Config, [[Destination, <<"0">>, <<"0">>, <<"0">>]]) + end. + +dead_letter_policy(Config) -> + [Server | _] = Servers = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + QQ = ?config(queue_name, Config), + CQ = <<"classic-dead_letter_policy">>, + ?assertEqual({'queue.declare_ok', QQ, 0, 0}, + declare(Ch, QQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), + ?assertEqual({'queue.declare_ok', CQ, 0, 0}, declare(Ch, CQ, [])), + ok = rabbit_ct_broker_helpers:set_policy( + Config, 0, <<"dlx">>, <<"dead_letter.*">>, <<"queues">>, + [{<<"dead-letter-exchange">>, <<"">>}, + {<<"dead-letter-routing-key">>, CQ}]), + RaName = ra_name(QQ), + test_dead_lettering(true, Config, Ch, Servers, RaName, QQ, CQ), + ok = rabbit_ct_broker_helpers:clear_policy(Config, 0, <<"dlx">>), + test_dead_lettering(false, Config, Ch, Servers, RaName, QQ, CQ). + +invalid_policy(Config) -> + [Server | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + QQ = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', QQ, 0, 0}, + declare(Ch, QQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), + ok = rabbit_ct_broker_helpers:set_policy( + Config, 0, <<"ha">>, <<"invalid_policy.*">>, <<"queues">>, + [{<<"ha-mode">>, <<"all">>}]), + ok = rabbit_ct_broker_helpers:set_policy( + Config, 0, <<"ttl">>, <<"invalid_policy.*">>, <<"queues">>, + [{<<"message-ttl">>, 5}]), + Info = rpc:call(Server, rabbit_quorum_queue, infos, + [rabbit_misc:r(<<"/">>, queue, QQ)]), + ?assertEqual('', proplists:get_value(policy, Info)), + ok = rabbit_ct_broker_helpers:clear_policy(Config, 0, <<"ha">>), + ok = rabbit_ct_broker_helpers:clear_policy(Config, 0, <<"ttl">>). + +dead_letter_to_quorum_queue(Config) -> + [Server | _] = Servers = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + QQ = ?config(queue_name, Config), + QQ2 = <<"dead_letter_to_quorum_queue-q2">>, + ?assertEqual({'queue.declare_ok', QQ, 0, 0}, + declare(Ch, QQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}, + {<<"x-dead-letter-exchange">>, longstr, <<>>}, + {<<"x-dead-letter-routing-key">>, longstr, QQ2} + ])), + ?assertEqual({'queue.declare_ok', QQ2, 0, 0}, + declare(Ch, QQ2, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), + RaName = ra_name(QQ), + RaName2 = ra_name(QQ2), + publish(Ch, QQ), + wait_for_messages_ready(Servers, RaName, 1), + wait_for_messages_pending_ack(Servers, RaName, 0), + wait_for_messages_ready(Servers, RaName2, 0), + wait_for_messages_pending_ack(Servers, RaName2, 0), + DeliveryTag = consume(Ch, QQ, false), + wait_for_messages_ready(Servers, RaName, 0), + wait_for_messages_pending_ack(Servers, RaName, 1), + wait_for_messages_ready(Servers, RaName2, 0), + wait_for_messages_pending_ack(Servers, RaName2, 0), + amqp_channel:cast(Ch, #'basic.nack'{delivery_tag = DeliveryTag, + multiple = false, + requeue = false}), + wait_for_messages_ready(Servers, RaName, 0), + wait_for_messages_pending_ack(Servers, RaName, 0), + wait_for_messages_ready(Servers, RaName2, 1), + wait_for_messages_pending_ack(Servers, RaName2, 0), + _ = consume(Ch, QQ2, false). + +dead_letter_from_classic_to_quorum_queue(Config) -> + [Server | _] = Servers = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + CQ = <<"classic-q-dead_letter_from_classic_to_quorum_queue">>, + QQ = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', CQ, 0, 0}, + declare(Ch, CQ, [{<<"x-dead-letter-exchange">>, longstr, <<>>}, + {<<"x-dead-letter-routing-key">>, longstr, QQ} + ])), + ?assertEqual({'queue.declare_ok', QQ, 0, 0}, + declare(Ch, QQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), + RaName = ra_name(QQ), + publish(Ch, CQ), + wait_for_messages_ready(Servers, RaName, 0), + wait_for_messages_pending_ack(Servers, RaName, 0), + wait_for_messages(Config, [[CQ, <<"1">>, <<"1">>, <<"0">>]]), + DeliveryTag = consume(Ch, CQ, false), + wait_for_messages_ready(Servers, RaName, 0), + wait_for_messages_pending_ack(Servers, RaName, 0), + wait_for_messages(Config, [[CQ, <<"1">>, <<"0">>, <<"1">>]]), + amqp_channel:cast(Ch, #'basic.nack'{delivery_tag = DeliveryTag, + multiple = false, + requeue = false}), + wait_for_messages_ready(Servers, RaName, 1), + wait_for_messages_pending_ack(Servers, RaName, 0), + wait_for_messages(Config, [[CQ, <<"0">>, <<"0">>, <<"0">>]]), + _ = consume(Ch, QQ, false), + rabbit_ct_client_helpers:close_channel(Ch). + +cleanup_queue_state_on_channel_after_publish(Config) -> + %% Declare/delete the queue in one channel and publish on a different one, + %% to verify that the cleanup is propagated through channels + [Server | _] = Servers = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + %% The stream coordinator is also a ra process, we need to ensure the quorum tests + %% are not affected by any other ra cluster that could be added in the future + Children = length(rpc:call(Server, supervisor, which_children, [ra_server_sup_sup])), + + Ch1 = rabbit_ct_client_helpers:open_channel(Config, Server), + Ch2 = rabbit_ct_client_helpers:open_channel(Config, Server), + QQ = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', QQ, 0, 0}, + declare(Ch1, QQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), + RaName = ra_name(QQ), + publish(Ch2, QQ), + Res = dirty_query(Servers, RaName, fun rabbit_fifo:query_consumer_count/1), + ct:pal ("Res ~p", [Res]), + wait_for_messages_pending_ack(Servers, RaName, 0), + wait_for_messages_ready(Servers, RaName, 1), + [NCh1, NCh2] = rpc:call(Server, rabbit_channel, list, []), + %% Check the channel state contains the state for the quorum queue on + %% channel 1 and 2 + wait_for_cleanup(Server, NCh1, 0), + wait_for_cleanup(Server, NCh2, 1), + %% then delete the queue and wait for the process to terminate + ?assertMatch(#'queue.delete_ok'{}, + amqp_channel:call(Ch1, #'queue.delete'{queue = QQ})), + wait_until(fun() -> + Children == length(rpc:call(Server, supervisor, which_children, + [ra_server_sup_sup])) + end), + %% Check that all queue states have been cleaned + wait_for_cleanup(Server, NCh2, 0), + wait_for_cleanup(Server, NCh1, 0). + +cleanup_queue_state_on_channel_after_subscribe(Config) -> + %% Declare/delete the queue and publish in one channel, while consuming on a + %% different one to verify that the cleanup is propagated through channels + [Server | _] = Servers = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + %% The stream coordinator is also a ra process, we need to ensure the quorum tests + %% are not affected by any other ra cluster that could be added in the future + Children = length(rpc:call(Server, supervisor, which_children, [ra_server_sup_sup])), + + Ch1 = rabbit_ct_client_helpers:open_channel(Config, Server), + Ch2 = rabbit_ct_client_helpers:open_channel(Config, Server), + QQ = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', QQ, 0, 0}, + declare(Ch1, QQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), + RaName = ra_name(QQ), + publish(Ch1, QQ), + wait_for_messages_ready(Servers, RaName, 1), + wait_for_messages_pending_ack(Servers, RaName, 0), + subscribe(Ch2, QQ, false), + receive + {#'basic.deliver'{delivery_tag = DeliveryTag, + redelivered = false}, _} -> + wait_for_messages_ready(Servers, RaName, 0), + wait_for_messages_pending_ack(Servers, RaName, 1), + amqp_channel:cast(Ch2, #'basic.ack'{delivery_tag = DeliveryTag, + multiple = true}), + wait_for_messages_ready(Servers, RaName, 0), + wait_for_messages_pending_ack(Servers, RaName, 0) + end, + [NCh1, NCh2] = rpc:call(Server, rabbit_channel, list, []), + %% Check the channel state contains the state for the quorum queue on channel 1 and 2 + wait_for_cleanup(Server, NCh1, 1), + wait_for_cleanup(Server, NCh2, 1), + ?assertMatch(#'queue.delete_ok'{}, amqp_channel:call(Ch1, #'queue.delete'{queue = QQ})), + wait_until(fun() -> + Children == length(rpc:call(Server, supervisor, which_children, [ra_server_sup_sup])) + end), + %% Check that all queue states have been cleaned + wait_for_cleanup(Server, NCh1, 0), + wait_for_cleanup(Server, NCh2, 0). + +recover_from_single_failure(Config) -> + [Server, Server1, Server2] = Servers = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + QQ = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', QQ, 0, 0}, + declare(Ch, QQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), + + ok = rabbit_ct_broker_helpers:stop_node(Config, Server2), + RaName = ra_name(QQ), + + publish(Ch, QQ), + publish(Ch, QQ), + publish(Ch, QQ), + wait_for_messages_ready([Server, Server1], RaName, 3), + wait_for_messages_pending_ack([Server, Server1], RaName, 0), + + ok = rabbit_ct_broker_helpers:start_node(Config, Server2), + wait_for_messages_ready(Servers, RaName, 3), + wait_for_messages_pending_ack(Servers, RaName, 0). + +recover_from_multiple_failures(Config) -> + [Server, Server1, Server2] = Servers = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + QQ = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', QQ, 0, 0}, + declare(Ch, QQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), + + ok = rabbit_ct_broker_helpers:stop_node(Config, Server1), + RaName = ra_name(QQ), + + publish(Ch, QQ), + publish(Ch, QQ), + publish(Ch, QQ), + + ok = rabbit_ct_broker_helpers:stop_node(Config, Server2), + + publish(Ch, QQ), + publish(Ch, QQ), + publish(Ch, QQ), + + wait_for_messages_ready([Server], RaName, 3), + wait_for_messages_pending_ack([Server], RaName, 0), + + ok = rabbit_ct_broker_helpers:start_node(Config, Server1), + ok = rabbit_ct_broker_helpers:start_node(Config, Server2), + + %% there is an assumption here that the messages were not lost and were + %% recovered when a quorum was restored. Not the best test perhaps. + wait_for_messages_ready(Servers, RaName, 6), + wait_for_messages_pending_ack(Servers, RaName, 0). + +publishing_to_unavailable_queue(Config) -> + %% publishing to an unavialable queue but with a reachable member should result + %% in the initial enqueuer session timing out and the message being nacked + [Server, Server1, Server2] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + TCh = rabbit_ct_client_helpers:open_channel(Config, Server), + QQ = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', QQ, 0, 0}, + declare(TCh, QQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), + + ok = rabbit_ct_broker_helpers:stop_node(Config, Server1), + ok = rabbit_ct_broker_helpers:stop_node(Config, Server2), + + ct:pal("opening channel to ~w", [Server]), + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + #'confirm.select_ok'{} = amqp_channel:call(Ch, #'confirm.select'{}), + amqp_channel:register_confirm_handler(Ch, self()), + publish_many(Ch, QQ, 1), + %% this should result in a nack + ok = receive + #'basic.ack'{} -> fail; + #'basic.nack'{} -> ok + after 90000 -> + exit(confirm_timeout) + end, + ok = rabbit_ct_broker_helpers:start_node(Config, Server1), + timer:sleep(2000), + publish_many(Ch, QQ, 1), + %% this should now be acked + ok = receive + #'basic.ack'{} -> ok; + #'basic.nack'{} -> fail + after 90000 -> + exit(confirm_timeout) + end, + %% check we get at least on ack + ok = rabbit_ct_broker_helpers:start_node(Config, Server2), + ok. + +leadership_takeover(Config) -> + %% Kill nodes in succession forcing the takeover of leadership, and all messages that + %% are in the queue. + [Server, Server1, Server2] = Servers = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + QQ = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', QQ, 0, 0}, + declare(Ch, QQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), + + ok = rabbit_ct_broker_helpers:stop_node(Config, Server1), + RaName = ra_name(QQ), + + publish(Ch, QQ), + publish(Ch, QQ), + publish(Ch, QQ), + + wait_for_messages_ready([Server], RaName, 3), + wait_for_messages_pending_ack([Server], RaName, 0), + + ok = rabbit_ct_broker_helpers:stop_node(Config, Server2), + + ok = rabbit_ct_broker_helpers:start_node(Config, Server1), + ok = rabbit_ct_broker_helpers:stop_node(Config, Server), + ok = rabbit_ct_broker_helpers:start_node(Config, Server2), + ok = rabbit_ct_broker_helpers:stop_node(Config, Server1), + ok = rabbit_ct_broker_helpers:start_node(Config, Server), + + wait_for_messages_ready([Server2, Server], RaName, 3), + wait_for_messages_pending_ack([Server2, Server], RaName, 0), + + ok = rabbit_ct_broker_helpers:start_node(Config, Server1), + wait_for_messages_ready(Servers, RaName, 3), + wait_for_messages_pending_ack(Servers, RaName, 0). + +metrics_cleanup_on_leadership_takeover(Config) -> + case is_mixed_versions() of + true -> + {skip, "metrics_cleanup_on_leadership_takeover tests isn't mixed version compatible"}; + false -> + metrics_cleanup_on_leadership_takeover0(Config) + end. + +metrics_cleanup_on_leadership_takeover0(Config) -> + %% Queue core metrics should be deleted from a node once the leadership is transferred + %% to another follower + [Server, _, _] = Servers = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + QQ = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', QQ, 0, 0}, + declare(Ch, QQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), + + RaName = ra_name(QQ), + publish(Ch, QQ), + publish(Ch, QQ), + publish(Ch, QQ), + + wait_for_messages_ready([Server], RaName, 3), + wait_for_messages_pending_ack([Server], RaName, 0), + {ok, _, {_, Leader}} = ra:members({RaName, Server}), + QRes = rabbit_misc:r(<<"/">>, queue, QQ), + wait_until( + fun() -> + case rpc:call(Leader, ets, lookup, [queue_coarse_metrics, QRes]) of + [{QRes, 3, 0, 3, _}] -> true; + _ -> false + end + end), + force_leader_change(Servers, QQ), + wait_until(fun () -> + [] =:= rpc:call(Leader, ets, lookup, [queue_coarse_metrics, QRes]) andalso + [] =:= rpc:call(Leader, ets, lookup, [queue_metrics, QRes]) + end), + ok. + +metrics_cleanup_on_leader_crash(Config) -> + %% Queue core metrics should be deleted from a node once the leadership is transferred + %% to another follower + [Server | _] = Servers = + rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + QQ = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', QQ, 0, 0}, + declare(Ch, QQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), + + RaName = ra_name(QQ), + publish(Ch, QQ), + publish(Ch, QQ), + publish(Ch, QQ), + + wait_for_messages_ready([Server], RaName, 3), + wait_for_messages_pending_ack([Server], RaName, 0), + {ok, _, {Name, Leader}} = ra:members({RaName, Server}), + QRes = rabbit_misc:r(<<"/">>, queue, QQ), + wait_until( + fun() -> + case rpc:call(Leader, ets, lookup, [queue_coarse_metrics, QRes]) of + [{QRes, 3, 0, 3, _}] -> true; + _ -> false + end + end), + Pid = rpc:call(Leader, erlang, whereis, [Name]), + rpc:call(Leader, erlang, exit, [Pid, kill]), + [Other | _] = lists:delete(Leader, Servers), + catch ra:trigger_election(Other), + %% kill it again just in case it came straight back up again + catch rpc:call(Leader, erlang, exit, [Pid, kill]), + + %% this isn't a reliable test as the leader can be restarted so quickly + %% after a crash it is elected leader of the next term as well. + wait_until( + fun() -> + [] == rpc:call(Leader, ets, lookup, [queue_coarse_metrics, QRes]) + end), + ok. + + +delete_declare(Config) -> + case is_mixed_versions() of + true -> + {skip, "delete_declare isn't mixed version reliable"}; + false -> + delete_declare0(Config) + end. + +delete_declare0(Config) -> + %% Delete cluster in ra is asynchronous, we have to ensure that we handle that in rmq + [Server | _] = Servers = rabbit_ct_broker_helpers:get_node_configs(Config, + nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + QQ = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', QQ, 0, 0}, + declare(Ch, QQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), + + RaName = ra_name(QQ), + publish(Ch, QQ), + publish(Ch, QQ), + publish(Ch, QQ), + wait_for_messages_ready(Servers, RaName, 3), + + ?assertMatch(#'queue.delete_ok'{}, + amqp_channel:call(Ch, #'queue.delete'{queue = QQ})), + %% the actual data deletions happen after the call has returned as a quorum + %% queue leader waits for all nodes to confirm they replicated the poison + %% pill before terminating itself. + case is_mixed_versions() of + true -> + %% when in mixed versions the QQ may not be able to apply the posion + %% pill for all nodes so need to wait longer for forced delete to + %% happen + timer:sleep(10000); + false -> + timer:sleep(1000) + end, + + ?assertEqual({'queue.declare_ok', QQ, 0, 0}, + declare(Ch, QQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), + + %% Ensure that is a new queue and it's empty + wait_for_messages_ready(Servers, RaName, 0), + wait_for_messages_pending_ack(Servers, RaName, 0). + +sync_queue(Config) -> + [Server | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + QQ = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', QQ, 0, 0}, + declare(Ch, QQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), + {error, _, _} = + rabbit_ct_broker_helpers:rabbitmqctl(Config, 0, [<<"sync_queue">>, QQ]), + ok. + +cancel_sync_queue(Config) -> + [Server | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + QQ = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', QQ, 0, 0}, + declare(Ch, QQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), + {error, _, _} = + rabbit_ct_broker_helpers:rabbitmqctl(Config, 0, [<<"cancel_sync_queue">>, QQ]), + ok. + +declare_during_node_down(Config) -> + [Server, DownServer, _] = Servers = rabbit_ct_broker_helpers:get_node_configs( + Config, nodename), + + stop_node(Config, DownServer), + % rabbit_ct_broker_helpers:stop_node(Config, DownServer), + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + QQ = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', QQ, 0, 0}, + declare(Ch, QQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), + + RaName = ra_name(QQ), + timer:sleep(2000), + rabbit_ct_broker_helpers:start_node(Config, DownServer), + publish(Ch, QQ), + wait_for_messages_ready(Servers, RaName, 1), + ok. + +simple_confirm_availability_on_leader_change(Config) -> + [Node1, Node2, _Node3] = + rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + %% declare a queue on node2 - this _should_ host the leader on node 2 + DCh = rabbit_ct_client_helpers:open_channel(Config, Node2), + QQ = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', QQ, 0, 0}, + declare(DCh, QQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), + + erlang:process_flag(trap_exit, true), + %% open a channel to another node + Ch = rabbit_ct_client_helpers:open_channel(Config, Node1), + #'confirm.select_ok'{} = amqp_channel:call(Ch, #'confirm.select'{}), + ok = publish_confirm(Ch, QQ), + + %% stop the node hosting the leader + ok = rabbit_ct_broker_helpers:stop_node(Config, Node2), + %% this should not fail as the channel should detect the new leader and + %% resend to that + ok = publish_confirm(Ch, QQ), + ok = rabbit_ct_broker_helpers:start_node(Config, Node2), + ok. + +confirm_availability_on_leader_change(Config) -> + [Node1, Node2, _Node3] = + rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + %% declare a queue on node2 - this _should_ host the leader on node 2 + DCh = rabbit_ct_client_helpers:open_channel(Config, Node2), + QQ = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', QQ, 0, 0}, + declare(DCh, QQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), + + erlang:process_flag(trap_exit, true), + Pid = spawn_link(fun () -> + %% open a channel to another node + Ch = rabbit_ct_client_helpers:open_channel(Config, Node1), + #'confirm.select_ok'{} = amqp_channel:call(Ch, #'confirm.select'{}), + ConfirmLoop = fun Loop() -> + ok = publish_confirm(Ch, QQ), + receive {done, P} -> + P ! done, + ok + after 0 -> Loop() end + end, + ConfirmLoop() + end), + + timer:sleep(500), + %% stop the node hosting the leader + stop_node(Config, Node2), + %% this should not fail as the channel should detect the new leader and + %% resend to that + timer:sleep(500), + Pid ! {done, self()}, + receive + done -> ok; + {'EXIT', Pid, Err} -> + exit(Err) + after 5500 -> + flush(100), + exit(bah) + end, + ok = rabbit_ct_broker_helpers:start_node(Config, Node2), + ok. + +flush(T) -> + receive X -> + ct:pal("flushed ~w", [X]), + flush(T) + after T -> + ok + end. + + +add_member_not_running(Config) -> + [Server | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + ct:pal("add_member_not_running config ~p", [Config]), + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + QQ = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', QQ, 0, 0}, + declare(Ch, QQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), + ?assertEqual({error, node_not_running}, + rpc:call(Server, rabbit_quorum_queue, add_member, + [<<"/">>, QQ, 'rabbit@burrow', 5000])). + +add_member_classic(Config) -> + [Server | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + CQ = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', CQ, 0, 0}, declare(Ch, CQ, [])), + ?assertEqual({error, classic_queue_not_supported}, + rpc:call(Server, rabbit_quorum_queue, add_member, + [<<"/">>, CQ, Server, 5000])). + +add_member_already_a_member(Config) -> + [Server | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + QQ = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', QQ, 0, 0}, + declare(Ch, QQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), + %% idempotent by design + ?assertEqual(ok, + rpc:call(Server, rabbit_quorum_queue, add_member, + [<<"/">>, QQ, Server, 5000])). + +add_member_not_found(Config) -> + [Server | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + QQ = ?config(queue_name, Config), + ?assertEqual({error, not_found}, + rpc:call(Server, rabbit_quorum_queue, add_member, + [<<"/">>, QQ, Server, 5000])). + +add_member(Config) -> + [Server0, Server1] = Servers0 = + rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + Ch = rabbit_ct_client_helpers:open_channel(Config, Server0), + QQ = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', QQ, 0, 0}, + declare(Ch, QQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), + ?assertEqual({error, node_not_running}, + rpc:call(Server0, rabbit_quorum_queue, add_member, + [<<"/">>, QQ, Server1, 5000])), + ok = rabbit_control_helper:command(stop_app, Server1), + ok = rabbit_control_helper:command(join_cluster, Server1, [atom_to_list(Server0)], []), + rabbit_control_helper:command(start_app, Server1), + ?assertEqual(ok, rpc:call(Server0, rabbit_quorum_queue, add_member, + [<<"/">>, QQ, Server1, 5000])), + Info = rpc:call(Server0, rabbit_quorum_queue, infos, + [rabbit_misc:r(<<"/">>, queue, QQ)]), + Servers = lists:sort(Servers0), + ?assertEqual(Servers, lists:sort(proplists:get_value(online, Info, []))). + +delete_member_not_running(Config) -> + [Server | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + QQ = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', QQ, 0, 0}, + declare(Ch, QQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), + %% it should be possible to delete members that are not online (e.g. decomissioned) + ?assertEqual(ok, + rpc:call(Server, rabbit_quorum_queue, delete_member, + [<<"/">>, QQ, 'rabbit@burrow'])). + +delete_member_classic(Config) -> + [Server | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + CQ = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', CQ, 0, 0}, declare(Ch, CQ, [])), + ?assertEqual({error, classic_queue_not_supported}, + rpc:call(Server, rabbit_quorum_queue, delete_member, + [<<"/">>, CQ, Server])). + +delete_member_queue_not_found(Config) -> + [Server | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + QQ = ?config(queue_name, Config), + ?assertEqual({error, not_found}, + rpc:call(Server, rabbit_quorum_queue, delete_member, + [<<"/">>, QQ, Server])). + +delete_member(Config) -> + [Server | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + QQ = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', QQ, 0, 0}, + declare(Ch, QQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), + timer:sleep(100), + ?assertEqual(ok, + rpc:call(Server, rabbit_quorum_queue, delete_member, + [<<"/">>, QQ, Server])). + +delete_member_not_a_member(Config) -> + [Server | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + QQ = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', QQ, 0, 0}, + declare(Ch, QQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), + timer:sleep(100), + ?assertEqual(ok, + rpc:call(Server, rabbit_quorum_queue, delete_member, + [<<"/">>, QQ, Server])), + %% idempotent by design + ?assertEqual(ok, + rpc:call(Server, rabbit_quorum_queue, delete_member, + [<<"/">>, QQ, Server])). + +delete_member_during_node_down(Config) -> + [Server, DownServer, Remove] = rabbit_ct_broker_helpers:get_node_configs( + Config, nodename), + + stop_node(Config, DownServer), + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + QQ = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', QQ, 0, 0}, + declare(Ch, QQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), + timer:sleep(200), + ?assertEqual(ok, rpc:call(Server, rabbit_quorum_queue, delete_member, + [<<"/">>, QQ, Remove])), + + rabbit_ct_broker_helpers:start_node(Config, DownServer), + ?assertEqual(ok, rpc:call(Server, rabbit_quorum_queue, repair_amqqueue_nodes, + [<<"/">>, QQ])), + ok. + +%% These tests check if node removal would cause any queues to lose (or not lose) +%% their quorum. See rabbitmq/rabbitmq-cli#389 for background. + +node_removal_is_quorum_critical(Config) -> + [Server | _] = Servers = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + QName = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', QName, 0, 0}, + declare(Ch, QName, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), + timer:sleep(100), + [begin + Qs = rpc:call(S, rabbit_quorum_queue, list_with_minimum_quorum, []), + ?assertEqual([QName], queue_names(Qs)) + end || S <- Servers]. + +node_removal_is_not_quorum_critical(Config) -> + [Server | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + QName = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', QName, 0, 0}, + declare(Ch, QName, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), + timer:sleep(100), + Qs = rpc:call(Server, rabbit_quorum_queue, list_with_minimum_quorum, []), + ?assertEqual([], Qs). + + +file_handle_reservations(Config) -> + case is_mixed_versions() of + true -> + {skip, "file_handle_reservations tests isn't mixed version compatible"}; + false -> + file_handle_reservations0(Config) + end. + +file_handle_reservations0(Config) -> + Servers = [Server1 | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + Ch = rabbit_ct_client_helpers:open_channel(Config, Server1), + QQ = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', QQ, 0, 0}, + declare(Ch, QQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), + RaName = ra_name(QQ), + {ok, _, {_, Leader}} = ra:members({RaName, Server1}), + [Follower1, Follower2] = Servers -- [Leader], + ?assertEqual([{files_reserved, 5}], + rpc:call(Leader, file_handle_cache, info, [[files_reserved]])), + ?assertEqual([{files_reserved, 2}], + rpc:call(Follower1, file_handle_cache, info, [[files_reserved]])), + ?assertEqual([{files_reserved, 2}], + rpc:call(Follower2, file_handle_cache, info, [[files_reserved]])), + force_leader_change(Servers, QQ), + {ok, _, {_, Leader0}} = ra:members({RaName, Server1}), + [Follower01, Follower02] = Servers -- [Leader0], + ?assertEqual([{files_reserved, 5}], + rpc:call(Leader0, file_handle_cache, info, [[files_reserved]])), + ?assertEqual([{files_reserved, 2}], + rpc:call(Follower01, file_handle_cache, info, [[files_reserved]])), + ?assertEqual([{files_reserved, 2}], + rpc:call(Follower02, file_handle_cache, info, [[files_reserved]])). + +file_handle_reservations_above_limit(Config) -> + [S1, S2, S3] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, S1), + QQ = ?config(queue_name, Config), + QQ2 = ?config(alt_queue_name, Config), + + Limit = rpc:call(S1, file_handle_cache, get_limit, []), + + ok = rpc:call(S1, file_handle_cache, set_limit, [3]), + ok = rpc:call(S2, file_handle_cache, set_limit, [3]), + ok = rpc:call(S3, file_handle_cache, set_limit, [3]), + + ?assertEqual({'queue.declare_ok', QQ, 0, 0}, + declare(Ch, QQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), + ?assertEqual({'queue.declare_ok', QQ2, 0, 0}, + declare(Ch, QQ2, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), + + ok = rpc:call(S1, file_handle_cache, set_limit, [Limit]), + ok = rpc:call(S2, file_handle_cache, set_limit, [Limit]), + ok = rpc:call(S3, file_handle_cache, set_limit, [Limit]). + +cleanup_data_dir(Config) -> + %% This test is slow, but also checks that we handle properly errors when + %% trying to delete a queue in minority. A case clause there had gone + %% previously unnoticed. + + [Server1, Server2] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + Ch = rabbit_ct_client_helpers:open_channel(Config, Server1), + QQ = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', QQ, 0, 0}, + declare(Ch, QQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), + timer:sleep(100), + + UId1 = proplists:get_value(ra_name(QQ), rpc:call(Server1, ra_directory, list_registered, [])), + UId2 = proplists:get_value(ra_name(QQ), rpc:call(Server2, ra_directory, list_registered, [])), + DataDir1 = rpc:call(Server1, ra_env, server_data_dir, [UId1]), + DataDir2 = rpc:call(Server2, ra_env, server_data_dir, [UId2]), + ?assert(filelib:is_dir(DataDir1)), + ?assert(filelib:is_dir(DataDir2)), + + ok = rabbit_ct_broker_helpers:stop_node(Config, Server2), + + ?assertMatch(#'queue.delete_ok'{}, + amqp_channel:call(Ch, #'queue.delete'{queue = QQ})), + ok = rabbit_ct_broker_helpers:stop_node(Config, Server2), + %% data dir 1 should be force deleted at this point + ?assert(not filelib:is_dir(DataDir1)), + ?assert(filelib:is_dir(DataDir2)), + ok = rabbit_ct_broker_helpers:start_node(Config, Server2), + timer:sleep(2000), + + ?assertEqual(ok, + rpc:call(Server2, rabbit_quorum_queue, cleanup_data_dir, [])), + ?assert(not filelib:is_dir(DataDir2)), + ok. + +reconnect_consumer_and_publish(Config) -> + [Server | _] = Servers = + rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + QQ = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', QQ, 0, 0}, + declare(Ch, QQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), + RaName = ra_name(QQ), + {ok, _, {_, Leader}} = ra:members({RaName, Server}), + [F1, F2] = lists:delete(Leader, Servers), + ChF = rabbit_ct_client_helpers:open_channel(Config, F1), + publish(Ch, QQ), + wait_for_messages_ready(Servers, RaName, 1), + wait_for_messages_pending_ack(Servers, RaName, 0), + subscribe(ChF, QQ, false), + receive + {#'basic.deliver'{redelivered = false}, _} -> + wait_for_messages_ready(Servers, RaName, 0), + wait_for_messages_pending_ack(Servers, RaName, 1) + end, + Up = [Leader, F2], + rabbit_ct_broker_helpers:block_traffic_between(F1, Leader), + rabbit_ct_broker_helpers:block_traffic_between(F1, F2), + wait_for_messages_ready(Up, RaName, 1), + wait_for_messages_pending_ack(Up, RaName, 0), + wait_for_messages_ready([F1], RaName, 0), + wait_for_messages_pending_ack([F1], RaName, 1), + rabbit_ct_broker_helpers:allow_traffic_between(F1, Leader), + rabbit_ct_broker_helpers:allow_traffic_between(F1, F2), + publish(Ch, QQ), + wait_for_messages_ready(Servers, RaName, 0), + wait_for_messages_pending_ack(Servers, RaName, 2), + receive + {#'basic.deliver'{delivery_tag = DeliveryTag, + redelivered = false}, _} -> + amqp_channel:cast(ChF, #'basic.ack'{delivery_tag = DeliveryTag, + multiple = false}), + wait_for_messages_ready(Servers, RaName, 0), + wait_for_messages_pending_ack(Servers, RaName, 1) + end, + receive + {#'basic.deliver'{delivery_tag = DeliveryTag2, + redelivered = true}, _} -> + amqp_channel:cast(ChF, #'basic.ack'{delivery_tag = DeliveryTag2, + multiple = false}), + wait_for_messages_ready(Servers, RaName, 0), + wait_for_messages_pending_ack(Servers, RaName, 0) + end. + +reconnect_consumer_and_wait(Config) -> + [Server | _] = Servers = + rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + QQ = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', QQ, 0, 0}, + declare(Ch, QQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), + RaName = ra_name(QQ), + {ok, _, {_, Leader}} = ra:members({RaName, Server}), + [F1, F2] = lists:delete(Leader, Servers), + ChF = rabbit_ct_client_helpers:open_channel(Config, F1), + publish(Ch, QQ), + wait_for_messages_ready(Servers, RaName, 1), + wait_for_messages_pending_ack(Servers, RaName, 0), + subscribe(ChF, QQ, false), + receive + {#'basic.deliver'{redelivered = false}, _} -> + wait_for_messages_ready(Servers, RaName, 0), + wait_for_messages_pending_ack(Servers, RaName, 1) + end, + Up = [Leader, F2], + rabbit_ct_broker_helpers:block_traffic_between(F1, Leader), + rabbit_ct_broker_helpers:block_traffic_between(F1, F2), + wait_for_messages_ready(Up, RaName, 1), + wait_for_messages_pending_ack(Up, RaName, 0), + wait_for_messages_ready([F1], RaName, 0), + wait_for_messages_pending_ack([F1], RaName, 1), + rabbit_ct_broker_helpers:allow_traffic_between(F1, Leader), + rabbit_ct_broker_helpers:allow_traffic_between(F1, F2), + wait_for_messages_ready(Servers, RaName, 0), + wait_for_messages_pending_ack(Servers, RaName, 1), + receive + {#'basic.deliver'{delivery_tag = DeliveryTag, + redelivered = true}, _} -> + amqp_channel:cast(ChF, #'basic.ack'{delivery_tag = DeliveryTag, + multiple = false}), + wait_for_messages_ready(Servers, RaName, 0), + wait_for_messages_pending_ack(Servers, RaName, 0) + end. + +reconnect_consumer_and_wait_channel_down(Config) -> + [Server | _] = Servers = + rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + QQ = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', QQ, 0, 0}, + declare(Ch, QQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), + RaName = ra_name(QQ), + {ok, _, {_, Leader}} = ra:members({RaName, Server}), + [F1, F2] = lists:delete(Leader, Servers), + ChF = rabbit_ct_client_helpers:open_channel(Config, F1), + publish(Ch, QQ), + wait_for_messages_ready(Servers, RaName, 1), + wait_for_messages_pending_ack(Servers, RaName, 0), + subscribe(ChF, QQ, false), + receive + {#'basic.deliver'{redelivered = false}, _} -> + wait_for_messages_ready(Servers, RaName, 0), + wait_for_messages_pending_ack(Servers, RaName, 1) + end, + Up = [Leader, F2], + rabbit_ct_broker_helpers:block_traffic_between(F1, Leader), + rabbit_ct_broker_helpers:block_traffic_between(F1, F2), + wait_for_messages_ready(Up, RaName, 1), + wait_for_messages_pending_ack(Up, RaName, 0), + wait_for_messages_ready([F1], RaName, 0), + wait_for_messages_pending_ack([F1], RaName, 1), + rabbit_ct_client_helpers:close_channel(ChF), + rabbit_ct_broker_helpers:allow_traffic_between(F1, Leader), + rabbit_ct_broker_helpers:allow_traffic_between(F1, F2), + %% Let's give it a few seconds to ensure it doesn't attempt to + %% deliver to the down channel - it shouldn't be monitored + %% at this time! + timer:sleep(5000), + wait_for_messages_ready(Servers, RaName, 1), + wait_for_messages_pending_ack(Servers, RaName, 0). + +delete_immediately_by_resource(Config) -> + Server = rabbit_ct_broker_helpers:get_node_config(Config, 0, nodename), + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + + %% The stream coordinator is also a ra process, we need to ensure the quorum tests + %% are not affected by any other ra cluster that could be added in the future + Children = length(rpc:call(Server, supervisor, which_children, [ra_server_sup_sup])), + + QQ = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', QQ, 0, 0}, + declare(Ch, QQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), + Cmd2 = ["eval", "rabbit_amqqueue:delete_immediately_by_resource([rabbit_misc:r(<<\"/\">>, queue, <<\"" ++ binary_to_list(QQ) ++ "\">>)])."], + ?assertEqual({ok, "ok\n"}, rabbit_ct_broker_helpers:rabbitmqctl(Config, 0, Cmd2)), + + %% Check that the application and process are down + wait_until(fun() -> + Children == length(rpc:call(Server, supervisor, which_children, [ra_server_sup_sup])) + end), + ?assertMatch({ra, _, _}, lists:keyfind(ra, 1, + rpc:call(Server, application, which_applications, []))). + +subscribe_redelivery_count(Config) -> + [Server | _] = Servers = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + QQ = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', QQ, 0, 0}, + declare(Ch, QQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), + + RaName = ra_name(QQ), + publish(Ch, QQ), + wait_for_messages_ready(Servers, RaName, 1), + wait_for_messages_pending_ack(Servers, RaName, 0), + subscribe(Ch, QQ, false), + + DCHeader = <<"x-delivery-count">>, + receive + {#'basic.deliver'{delivery_tag = DeliveryTag, + redelivered = false}, + #amqp_msg{props = #'P_basic'{headers = H0}}} -> + ?assertMatch(undefined, rabbit_basic:header(DCHeader, H0)), + amqp_channel:cast(Ch, #'basic.nack'{delivery_tag = DeliveryTag, + multiple = false, + requeue = true}) + after 5000 -> + exit(basic_deliver_timeout) + end, + + receive + {#'basic.deliver'{delivery_tag = DeliveryTag1, + redelivered = true}, + #amqp_msg{props = #'P_basic'{headers = H1}}} -> + ?assertMatch({DCHeader, _, 1}, rabbit_basic:header(DCHeader, H1)), + amqp_channel:cast(Ch, #'basic.nack'{delivery_tag = DeliveryTag1, + multiple = false, + requeue = true}) + after 5000 -> + exit(basic_deliver_timeout_2) + end, + + receive + {#'basic.deliver'{delivery_tag = DeliveryTag2, + redelivered = true}, + #amqp_msg{props = #'P_basic'{headers = H2}}} -> + ?assertMatch({DCHeader, _, 2}, rabbit_basic:header(DCHeader, H2)), + amqp_channel:cast(Ch, #'basic.ack'{delivery_tag = DeliveryTag2, + multiple = false}), + ct:pal("wait_for_messages_ready", []), + wait_for_messages_ready(Servers, RaName, 0), + ct:pal("wait_for_messages_pending_ack", []), + wait_for_messages_pending_ack(Servers, RaName, 0) + after 5000 -> + flush(500), + exit(basic_deliver_timeout_3) + end. + +subscribe_redelivery_limit(Config) -> + [Server | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + QQ = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', QQ, 0, 0}, + declare(Ch, QQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}, + {<<"x-delivery-limit">>, long, 1}])), + + publish(Ch, QQ), + wait_for_messages(Config, [[QQ, <<"1">>, <<"1">>, <<"0">>]]), + subscribe(Ch, QQ, false), + + DCHeader = <<"x-delivery-count">>, + receive + {#'basic.deliver'{delivery_tag = DeliveryTag, + redelivered = false}, + #amqp_msg{props = #'P_basic'{headers = H0}}} -> + ?assertMatch(undefined, rabbit_basic:header(DCHeader, H0)), + amqp_channel:cast(Ch, #'basic.nack'{delivery_tag = DeliveryTag, + multiple = false, + requeue = true}) + end, + + wait_for_messages(Config, [[QQ, <<"1">>, <<"0">>, <<"1">>]]), + receive + {#'basic.deliver'{delivery_tag = DeliveryTag1, + redelivered = true}, + #amqp_msg{props = #'P_basic'{headers = H1}}} -> + ?assertMatch({DCHeader, _, 1}, rabbit_basic:header(DCHeader, H1)), + amqp_channel:cast(Ch, #'basic.nack'{delivery_tag = DeliveryTag1, + multiple = false, + requeue = true}) + end, + + wait_for_messages(Config, [[QQ, <<"0">>, <<"0">>, <<"0">>]]), + receive + {#'basic.deliver'{redelivered = true}, #amqp_msg{}} -> + throw(unexpected_redelivery) + after 2000 -> + ok + end. + +subscribe_redelivery_policy(Config) -> + [Server | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + QQ = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', QQ, 0, 0}, + declare(Ch, QQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), + + ok = rabbit_ct_broker_helpers:set_policy( + Config, 0, <<"delivery-limit">>, <<".*">>, <<"queues">>, + [{<<"delivery-limit">>, 1}]), + + publish(Ch, QQ), + wait_for_messages(Config, [[QQ, <<"1">>, <<"1">>, <<"0">>]]), + subscribe(Ch, QQ, false), + + DCHeader = <<"x-delivery-count">>, + receive + {#'basic.deliver'{delivery_tag = DeliveryTag, + redelivered = false}, + #amqp_msg{props = #'P_basic'{headers = H0}}} -> + ?assertMatch(undefined, rabbit_basic:header(DCHeader, H0)), + amqp_channel:cast(Ch, #'basic.nack'{delivery_tag = DeliveryTag, + multiple = false, + requeue = true}) + end, + + wait_for_messages(Config, [[QQ, <<"1">>, <<"0">>, <<"1">>]]), + receive + {#'basic.deliver'{delivery_tag = DeliveryTag1, + redelivered = true}, + #amqp_msg{props = #'P_basic'{headers = H1}}} -> + ?assertMatch({DCHeader, _, 1}, rabbit_basic:header(DCHeader, H1)), + amqp_channel:cast(Ch, #'basic.nack'{delivery_tag = DeliveryTag1, + multiple = false, + requeue = true}) + end, + + wait_for_messages(Config, [[QQ, <<"0">>, <<"0">>, <<"0">>]]), + receive + {#'basic.deliver'{redelivered = true}, #amqp_msg{}} -> + throw(unexpected_redelivery) + after 2000 -> + ok + end, + ok = rabbit_ct_broker_helpers:clear_policy(Config, 0, <<"delivery-limit">>). + +subscribe_redelivery_limit_with_dead_letter(Config) -> + [Server | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + QQ = ?config(queue_name, Config), + DLX = <<"subcribe_redelivery_limit_with_dead_letter_dlx">>, + ?assertEqual({'queue.declare_ok', QQ, 0, 0}, + declare(Ch, QQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}, + {<<"x-delivery-limit">>, long, 1}, + {<<"x-dead-letter-exchange">>, longstr, <<>>}, + {<<"x-dead-letter-routing-key">>, longstr, DLX} + ])), + ?assertEqual({'queue.declare_ok', DLX, 0, 0}, + declare(Ch, DLX, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), + + publish(Ch, QQ), + wait_for_messages(Config, [[QQ, <<"1">>, <<"1">>, <<"0">>]]), + subscribe(Ch, QQ, false), + + DCHeader = <<"x-delivery-count">>, + receive + {#'basic.deliver'{delivery_tag = DeliveryTag, + redelivered = false}, + #amqp_msg{props = #'P_basic'{headers = H0}}} -> + ?assertMatch(undefined, rabbit_basic:header(DCHeader, H0)), + amqp_channel:cast(Ch, #'basic.nack'{delivery_tag = DeliveryTag, + multiple = false, + requeue = true}) + end, + + wait_for_messages(Config, [[QQ, <<"1">>, <<"0">>, <<"1">>]]), + receive + {#'basic.deliver'{delivery_tag = DeliveryTag1, + redelivered = true}, + #amqp_msg{props = #'P_basic'{headers = H1}}} -> + ?assertMatch({DCHeader, _, 1}, rabbit_basic:header(DCHeader, H1)), + amqp_channel:cast(Ch, #'basic.nack'{delivery_tag = DeliveryTag1, + multiple = false, + requeue = true}) + end, + + wait_for_messages(Config, [[QQ, <<"0">>, <<"0">>, <<"0">>]]), + wait_for_messages(Config, [[DLX, <<"1">>, <<"1">>, <<"0">>]]). + +consume_redelivery_count(Config) -> + [Server | _] = Servers = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + QQ = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', QQ, 0, 0}, + declare(Ch, QQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), + RaName = ra_name(QQ), + publish(Ch, QQ), + wait_for_messages_ready(Servers, RaName, 1), + wait_for_messages_pending_ack(Servers, RaName, 0), + + DCHeader = <<"x-delivery-count">>, + + {#'basic.get_ok'{delivery_tag = DeliveryTag, + redelivered = false}, + #amqp_msg{props = #'P_basic'{headers = H0}}} = + amqp_channel:call(Ch, #'basic.get'{queue = QQ, + no_ack = false}), + ?assertMatch({DCHeader, _, 0}, rabbit_basic:header(DCHeader, H0)), + amqp_channel:cast(Ch, #'basic.nack'{delivery_tag = DeliveryTag, + multiple = false, + requeue = true}), + %% wait for requeuing + timer:sleep(500), + + {#'basic.get_ok'{delivery_tag = DeliveryTag1, + redelivered = true}, + #amqp_msg{props = #'P_basic'{headers = H1}}} = + amqp_channel:call(Ch, #'basic.get'{queue = QQ, + no_ack = false}), + ?assertMatch({DCHeader, _, 1}, rabbit_basic:header(DCHeader, H1)), + amqp_channel:cast(Ch, #'basic.nack'{delivery_tag = DeliveryTag1, + multiple = false, + requeue = true}), + + {#'basic.get_ok'{delivery_tag = DeliveryTag2, + redelivered = true}, + #amqp_msg{props = #'P_basic'{headers = H2}}} = + amqp_channel:call(Ch, #'basic.get'{queue = QQ, + no_ack = false}), + ?assertMatch({DCHeader, _, 2}, rabbit_basic:header(DCHeader, H2)), + amqp_channel:cast(Ch, #'basic.nack'{delivery_tag = DeliveryTag2, + multiple = false, + requeue = true}), + ok. + +message_bytes_metrics(Config) -> + [Server | _] = Servers = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + QQ = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', QQ, 0, 0}, + declare(Ch, QQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), + + RaName = ra_name(QQ), + {ok, _, {_, Leader}} = ra:members({RaName, Server}), + QRes = rabbit_misc:r(<<"/">>, queue, QQ), + + publish(Ch, QQ), + + wait_for_messages_ready(Servers, RaName, 1), + wait_for_messages_pending_ack(Servers, RaName, 0), + wait_until(fun() -> + {3, 3, 0} == get_message_bytes(Leader, QRes) + end), + + subscribe(Ch, QQ, false), + + wait_for_messages_ready(Servers, RaName, 0), + wait_for_messages_pending_ack(Servers, RaName, 1), + wait_until(fun() -> + {3, 0, 3} == get_message_bytes(Leader, QRes) + end), + + receive + {#'basic.deliver'{delivery_tag = DeliveryTag, + redelivered = false}, _} -> + amqp_channel:cast(Ch, #'basic.nack'{delivery_tag = DeliveryTag, + multiple = false, + requeue = false}), + wait_for_messages_ready(Servers, RaName, 0), + wait_for_messages_pending_ack(Servers, RaName, 0), + wait_until(fun() -> + {0, 0, 0} == get_message_bytes(Leader, QRes) + end) + end, + + %% Let's publish and then close the consumer channel. Messages must be + %% returned to the queue + publish(Ch, QQ), + + wait_for_messages_ready(Servers, RaName, 0), + wait_for_messages_pending_ack(Servers, RaName, 1), + wait_until(fun() -> + {3, 0, 3} == get_message_bytes(Leader, QRes) + end), + + rabbit_ct_client_helpers:close_channel(Ch), + + wait_for_messages_ready(Servers, RaName, 1), + wait_for_messages_pending_ack(Servers, RaName, 0), + wait_until(fun() -> + {3, 3, 0} == get_message_bytes(Leader, QRes) + end), + ok. + +memory_alarm_rolls_wal(Config) -> + [Server | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + WalDataDir = rpc:call(Server, ra_env, wal_data_dir, []), + [Wal0] = filelib:wildcard(WalDataDir ++ "/*.wal"), + rabbit_ct_broker_helpers:set_alarm(Config, Server, memory), + rabbit_ct_helpers:await_condition( + fun() -> rabbit_ct_broker_helpers:get_alarms(Config, Server) =/= [] end + ), + timer:sleep(1000), + [Wal1] = filelib:wildcard(WalDataDir ++ "/*.wal"), + ?assert(Wal0 =/= Wal1), + %% roll over shouldn't happen if we trigger a new alarm in less than + %% min_wal_roll_over_interval + rabbit_ct_broker_helpers:set_alarm(Config, Server, memory), + rabbit_ct_helpers:await_condition( + fun() -> rabbit_ct_broker_helpers:get_alarms(Config, Server) =/= [] end + ), + timer:sleep(1000), + [Wal2] = filelib:wildcard(WalDataDir ++ "/*.wal"), + ?assert(Wal1 == Wal2), + ok = rpc:call(Server, rabbit_alarm, clear_alarm, + [{{resource_limit, memory, Server}, []}]), + timer:sleep(1000), + ok. + +queue_length_limit_drop_head(Config) -> + [Server | _] = Servers = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + QQ = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', QQ, 0, 0}, + declare(Ch, QQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}, + {<<"x-max-length">>, long, 1}])), + + RaName = ra_name(QQ), + ok = amqp_channel:cast(Ch, + #'basic.publish'{routing_key = QQ}, + #amqp_msg{props = #'P_basic'{delivery_mode = 2}, + payload = <<"msg1">>}), + ok = amqp_channel:cast(Ch, + #'basic.publish'{routing_key = QQ}, + #amqp_msg{props = #'P_basic'{delivery_mode = 2}, + payload = <<"msg2">>}), + wait_for_consensus(QQ, Config), + wait_for_messages_ready(Servers, RaName, 1), + wait_for_messages_pending_ack(Servers, RaName, 0), + wait_for_messages_total(Servers, RaName, 1), + ?assertMatch({#'basic.get_ok'{}, #amqp_msg{payload = <<"msg2">>}}, + amqp_channel:call(Ch, #'basic.get'{queue = QQ, + no_ack = true})). + +queue_length_limit_reject_publish(Config) -> + [Server | _] = Servers = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + QQ = ?config(queue_name, Config), + RaName = ra_name(QQ), + ?assertEqual({'queue.declare_ok', QQ, 0, 0}, + declare(Ch, QQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}, + {<<"x-max-length">>, long, 1}, + {<<"x-overflow">>, longstr, <<"reject-publish">>}])), + + #'confirm.select_ok'{} = amqp_channel:call(Ch, #'confirm.select'{}), + ok = publish_confirm(Ch, QQ), + ok = publish_confirm(Ch, QQ), + %% give the channel some time to process the async reject_publish notification + %% now that we are over the limit it should start failing + wait_for_messages_total(Servers, RaName, 2), + fail = publish_confirm(Ch, QQ), + %% remove all messages + ?assertMatch({#'basic.get_ok'{}, #amqp_msg{payload = _}}, + amqp_channel:call(Ch, #'basic.get'{queue = QQ, + no_ack = true})), + ?assertMatch({#'basic.get_ok'{}, #amqp_msg{payload = _}}, + amqp_channel:call(Ch, #'basic.get'{queue = QQ, + no_ack = true})), + %% publish should be allowed again now + ok = publish_confirm(Ch, QQ), + ok. + +queue_length_in_memory_limit_basic_get(Config) -> + [Server | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + QQ = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', QQ, 0, 0}, + declare(Ch, QQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}, + {<<"x-max-in-memory-length">>, long, 1}])), + + RaName = ra_name(QQ), + Msg1 = <<"msg1">>, + ok = amqp_channel:cast(Ch, + #'basic.publish'{routing_key = QQ}, + #amqp_msg{props = #'P_basic'{delivery_mode = 2}, + payload = Msg1}), + ok = amqp_channel:cast(Ch, + #'basic.publish'{routing_key = QQ}, + #amqp_msg{props = #'P_basic'{delivery_mode = 2}, + payload = <<"msg2">>}), + + wait_for_messages(Config, [[QQ, <<"2">>, <<"2">>, <<"0">>]]), + + ?assertEqual([{1, byte_size(Msg1)}], + dirty_query([Server], RaName, fun rabbit_fifo:query_in_memory_usage/1)), + + ?assertMatch({#'basic.get_ok'{}, #amqp_msg{payload = Msg1}}, + amqp_channel:call(Ch, #'basic.get'{queue = QQ, + no_ack = true})), + ?assertMatch({#'basic.get_ok'{}, #amqp_msg{payload = <<"msg2">>}}, + amqp_channel:call(Ch, #'basic.get'{queue = QQ, + no_ack = true})). + +queue_length_in_memory_limit_subscribe(Config) -> + [Server | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + QQ = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', QQ, 0, 0}, + declare(Ch, QQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}, + {<<"x-max-in-memory-length">>, long, 1}])), + + RaName = ra_name(QQ), + Msg1 = <<"msg1">>, + Msg2 = <<"msg11">>, + publish(Ch, QQ, Msg1), + publish(Ch, QQ, Msg2), + wait_for_messages(Config, [[QQ, <<"2">>, <<"2">>, <<"0">>]]), + + ?assertEqual([{1, byte_size(Msg1)}], + dirty_query([Server], RaName, fun rabbit_fifo:query_in_memory_usage/1)), + + subscribe(Ch, QQ, false), + receive + {#'basic.deliver'{delivery_tag = DeliveryTag1, + redelivered = false}, + #amqp_msg{payload = Msg1}} -> + amqp_channel:cast(Ch, #'basic.ack'{delivery_tag = DeliveryTag1, + multiple = false}) + end, + ?assertEqual([{0, 0}], + dirty_query([Server], RaName, fun rabbit_fifo:query_in_memory_usage/1)), + receive + {#'basic.deliver'{delivery_tag = DeliveryTag2, + redelivered = false}, + #amqp_msg{payload = Msg2}} -> + amqp_channel:cast(Ch, #'basic.ack'{delivery_tag = DeliveryTag2, + multiple = false}) + end. + +queue_length_in_memory_limit(Config) -> + [Server | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + QQ = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', QQ, 0, 0}, + declare(Ch, QQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}, + {<<"x-max-in-memory-length">>, long, 2}])), + + RaName = ra_name(QQ), + Msg1 = <<"msg1">>, + Msg2 = <<"msg11">>, + Msg3 = <<"msg111">>, + Msg4 = <<"msg1111">>, + + publish(Ch, QQ, Msg1), + publish(Ch, QQ, Msg2), + publish(Ch, QQ, Msg3), + wait_for_messages(Config, [[QQ, <<"3">>, <<"3">>, <<"0">>]]), + + ?assertEqual([{2, byte_size(Msg1) + byte_size(Msg2)}], + dirty_query([Server], RaName, fun rabbit_fifo:query_in_memory_usage/1)), + + ?assertMatch({#'basic.get_ok'{}, #amqp_msg{payload = Msg1}}, + amqp_channel:call(Ch, #'basic.get'{queue = QQ, + no_ack = true})), + + wait_for_messages(Config, [[QQ, <<"2">>, <<"2">>, <<"0">>]]), + publish(Ch, QQ, Msg4), + wait_for_messages(Config, [[QQ, <<"3">>, <<"3">>, <<"0">>]]), + + ?assertEqual([{2, byte_size(Msg2) + byte_size(Msg4)}], + dirty_query([Server], RaName, fun rabbit_fifo:query_in_memory_usage/1)). + +queue_length_in_memory_limit_returns(Config) -> + [Server | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + QQ = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', QQ, 0, 0}, + declare(Ch, QQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}, + {<<"x-max-in-memory-length">>, long, 2}])), + + RaName = ra_name(QQ), + Msg1 = <<"msg1">>, + Msg2 = <<"msg11">>, + Msg3 = <<"msg111">>, + Msg4 = <<"msg111">>, + publish(Ch, QQ, Msg1), + publish(Ch, QQ, Msg2), + wait_for_messages(Config, [[QQ, <<"2">>, <<"2">>, <<"0">>]]), + + ?assertEqual([{2, byte_size(Msg1) + byte_size(Msg2)}], + dirty_query([Server], RaName, fun rabbit_fifo:query_in_memory_usage/1)), + + ?assertMatch({#'basic.get_ok'{}, #amqp_msg{payload = Msg1}}, + amqp_channel:call(Ch, #'basic.get'{queue = QQ, + no_ack = false})), + + {#'basic.get_ok'{delivery_tag = DTag2}, #amqp_msg{payload = Msg2}} = + amqp_channel:call(Ch, #'basic.get'{queue = QQ, + no_ack = false}), + + publish(Ch, QQ, Msg3), + publish(Ch, QQ, Msg4), + + %% Ensure that returns are subject to in memory limits too + wait_for_messages(Config, [[QQ, <<"4">>, <<"2">>, <<"2">>]]), + amqp_channel:cast(Ch, #'basic.nack'{delivery_tag = DTag2, + multiple = true, + requeue = true}), + wait_for_messages(Config, [[QQ, <<"4">>, <<"4">>, <<"0">>]]), + + ?assertEqual([{2, byte_size(Msg3) + byte_size(Msg4)}], + dirty_query([Server], RaName, fun rabbit_fifo:query_in_memory_usage/1)). + +queue_length_in_memory_bytes_limit_basic_get(Config) -> + [Server | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + QQ = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', QQ, 0, 0}, + declare(Ch, QQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}, + {<<"x-max-in-memory-bytes">>, long, 6}])), + + RaName = ra_name(QQ), + Msg1 = <<"msg1">>, + ok = amqp_channel:cast(Ch, + #'basic.publish'{routing_key = QQ}, + #amqp_msg{props = #'P_basic'{delivery_mode = 2}, + payload = Msg1}), + ok = amqp_channel:cast(Ch, + #'basic.publish'{routing_key = QQ}, + #amqp_msg{props = #'P_basic'{delivery_mode = 2}, + payload = <<"msg2">>}), + + wait_for_messages(Config, [[QQ, <<"2">>, <<"2">>, <<"0">>]]), + + ?assertEqual([{1, byte_size(Msg1)}], + dirty_query([Server], RaName, fun rabbit_fifo:query_in_memory_usage/1)), + + ?assertMatch({#'basic.get_ok'{}, #amqp_msg{payload = Msg1}}, + amqp_channel:call(Ch, #'basic.get'{queue = QQ, + no_ack = true})), + ?assertMatch({#'basic.get_ok'{}, #amqp_msg{payload = <<"msg2">>}}, + amqp_channel:call(Ch, #'basic.get'{queue = QQ, + no_ack = true})). + +queue_length_in_memory_bytes_limit_subscribe(Config) -> + [Server | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + QQ = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', QQ, 0, 0}, + declare(Ch, QQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}, + {<<"x-max-in-memory-bytes">>, long, 6}])), + + RaName = ra_name(QQ), + Msg1 = <<"msg1">>, + Msg2 = <<"msg11">>, + publish(Ch, QQ, Msg1), + publish(Ch, QQ, Msg2), + wait_for_messages(Config, [[QQ, <<"2">>, <<"2">>, <<"0">>]]), + + ?assertEqual([{1, byte_size(Msg1)}], + dirty_query([Server], RaName, fun rabbit_fifo:query_in_memory_usage/1)), + + subscribe(Ch, QQ, false), + receive + {#'basic.deliver'{delivery_tag = DeliveryTag1, + redelivered = false}, + #amqp_msg{payload = Msg1}} -> + amqp_channel:cast(Ch, #'basic.ack'{delivery_tag = DeliveryTag1, + multiple = false}) + end, + ?assertEqual([{0, 0}], + dirty_query([Server], RaName, fun rabbit_fifo:query_in_memory_usage/1)), + receive + {#'basic.deliver'{delivery_tag = DeliveryTag2, + redelivered = false}, + #amqp_msg{payload = Msg2}} -> + amqp_channel:cast(Ch, #'basic.ack'{delivery_tag = DeliveryTag2, + multiple = false}) + end. + +queue_length_in_memory_bytes_limit(Config) -> + [Server | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + QQ = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', QQ, 0, 0}, + declare(Ch, QQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}, + {<<"x-max-in-memory-bytes">>, long, 12}])), + + RaName = ra_name(QQ), + Msg1 = <<"msg1">>, + Msg2 = <<"msg11">>, + Msg3 = <<"msg111">>, + Msg4 = <<"msg1111">>, + + publish(Ch, QQ, Msg1), + publish(Ch, QQ, Msg2), + publish(Ch, QQ, Msg3), + wait_for_messages(Config, [[QQ, <<"3">>, <<"3">>, <<"0">>]]), + + ?assertEqual([{2, byte_size(Msg1) + byte_size(Msg2)}], + dirty_query([Server], RaName, fun rabbit_fifo:query_in_memory_usage/1)), + + ?assertMatch({#'basic.get_ok'{}, #amqp_msg{payload = Msg1}}, + amqp_channel:call(Ch, #'basic.get'{queue = QQ, + no_ack = true})), + + wait_for_messages(Config, [[QQ, <<"2">>, <<"2">>, <<"0">>]]), + publish(Ch, QQ, Msg4), + wait_for_messages(Config, [[QQ, <<"3">>, <<"3">>, <<"0">>]]), + + ?assertEqual([{2, byte_size(Msg2) + byte_size(Msg4)}], + dirty_query([Server], RaName, fun rabbit_fifo:query_in_memory_usage/1)). + +queue_length_in_memory_purge(Config) -> + [Server | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + QQ = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', QQ, 0, 0}, + declare(Ch, QQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}, + {<<"x-max-in-memory-length">>, long, 2}])), + + RaName = ra_name(QQ), + Msg1 = <<"msg1">>, + Msg2 = <<"msg11">>, + Msg3 = <<"msg111">>, + + publish(Ch, QQ, Msg1), + publish(Ch, QQ, Msg2), + publish(Ch, QQ, Msg3), + wait_for_messages(Config, [[QQ, <<"3">>, <<"3">>, <<"0">>]]), + + ?assertEqual([{2, byte_size(Msg1) + byte_size(Msg2)}], + dirty_query([Server], RaName, fun rabbit_fifo:query_in_memory_usage/1)), + + {'queue.purge_ok', 3} = amqp_channel:call(Ch, #'queue.purge'{queue = QQ}), + + ?assertEqual([{0, 0}], + dirty_query([Server], RaName, fun rabbit_fifo:query_in_memory_usage/1)). + +peek(Config) -> + [Server | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + QQ = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', QQ, 0, 0}, + declare(Ch, QQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}, + {<<"x-max-in-memory-length">>, long, 2}])), + + Msg1 = <<"msg1">>, + Msg2 = <<"msg11">>, + + QName = rabbit_misc:r(<<"/">>, queue, QQ), + ?assertMatch({error, no_message_at_pos}, + rabbit_ct_broker_helpers:rpc(Config, 0, rabbit_quorum_queue, + peek, [1, QName])), + publish(Ch, QQ, Msg1), + publish(Ch, QQ, Msg2), + wait_for_messages(Config, [[QQ, <<"2">>, <<"2">>, <<"0">>]]), + + ?assertMatch({ok, [_|_]}, + rabbit_ct_broker_helpers:rpc(Config, 0, rabbit_quorum_queue, + peek, [1, QName])), + ?assertMatch({ok, [_|_]}, + rabbit_ct_broker_helpers:rpc(Config, 0, rabbit_quorum_queue, + peek, [2, QName])), + ?assertMatch({error, no_message_at_pos}, + rabbit_ct_broker_helpers:rpc(Config, 0, rabbit_quorum_queue, + peek, [3, QName])), + + wait_for_messages(Config, [[QQ, <<"2">>, <<"2">>, <<"0">>]]), + ok. + +in_memory(Config) -> + [Server | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + QQ = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', QQ, 0, 0}, + declare(Ch, QQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), + + RaName = ra_name(QQ), + Msg1 = <<"msg1">>, + Msg2 = <<"msg11">>, + + publish(Ch, QQ, Msg1), + + wait_for_messages(Config, [[QQ, <<"1">>, <<"1">>, <<"0">>]]), + ?assertEqual([{1, byte_size(Msg1)}], + dirty_query([Server], RaName, fun rabbit_fifo:query_in_memory_usage/1)), + + subscribe(Ch, QQ, false), + + wait_for_messages(Config, [[QQ, <<"1">>, <<"0">>, <<"1">>]]), + ?assertEqual([{0, 0}], + dirty_query([Server], RaName, fun rabbit_fifo:query_in_memory_usage/1)), + + publish(Ch, QQ, Msg2), + + wait_for_messages(Config, [[QQ, <<"2">>, <<"0">>, <<"2">>]]), + ?assertEqual([{0, 0}], + dirty_query([Server], RaName, fun rabbit_fifo:query_in_memory_usage/1)), + + receive + {#'basic.deliver'{delivery_tag = DeliveryTag}, #amqp_msg{}} -> + amqp_channel:cast(Ch, #'basic.ack'{delivery_tag = DeliveryTag, + multiple = false}) + end, + + wait_for_messages(Config, [[QQ, <<"1">>, <<"0">>, <<"1">>]]), + ?assertEqual([{0, 0}], + dirty_query([Server], RaName, fun rabbit_fifo:query_in_memory_usage/1)). + +consumer_metrics(Config) -> + [Server | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch1 = rabbit_ct_client_helpers:open_channel(Config, Server), + QQ = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', QQ, 0, 0}, + declare(Ch1, QQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), + subscribe(Ch1, QQ, false), + + RaName = ra_name(QQ), + {ok, _, {_, Leader}} = ra:members({RaName, Server}), + timer:sleep(5000), + QNameRes = rabbit_misc:r(<<"/">>, queue, QQ), + [{_, PropList, _}] = rpc:call(Leader, ets, lookup, [queue_metrics, QNameRes]), + ?assertMatch([{consumers, 1}], lists:filter(fun({Key, _}) -> + Key == consumers + end, PropList)). + +delete_if_empty(Config) -> + Server = rabbit_ct_broker_helpers:get_node_config(Config, 0, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + QQ = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', QQ, 0, 0}, + declare(Ch, QQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), + publish(Ch, QQ), + wait_for_messages(Config, [[QQ, <<"1">>, <<"1">>, <<"0">>]]), + %% Try to delete the quorum queue + ?assertExit({{shutdown, {connection_closing, {server_initiated_close, 540, _}}}, _}, + amqp_channel:call(Ch, #'queue.delete'{queue = QQ, + if_empty = true})). + +delete_if_unused(Config) -> + Server = rabbit_ct_broker_helpers:get_node_config(Config, 0, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + QQ = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', QQ, 0, 0}, + declare(Ch, QQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), + publish(Ch, QQ), + wait_for_messages(Config, [[QQ, <<"1">>, <<"1">>, <<"0">>]]), + %% Try to delete the quorum queue + ?assertExit({{shutdown, {connection_closing, {server_initiated_close, 540, _}}}, _}, + amqp_channel:call(Ch, #'queue.delete'{queue = QQ, + if_unused = true})). + +queue_ttl(Config) -> + Server = rabbit_ct_broker_helpers:get_node_config(Config, 0, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + QQ = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', QQ, 0, 0}, + declare(Ch, QQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}, + {<<"x-expires">>, long, 1000}])), + timer:sleep(5500), + %% check queue no longer exists + ?assertExit( + {{shutdown, + {server_initiated_close,404, + <<"NOT_FOUND - no queue 'queue_ttl' in vhost '/'">>}}, + _}, + amqp_channel:call(Ch, #'queue.declare'{queue = QQ, + passive = true, + durable = true, + auto_delete = false, + arguments = [{<<"x-queue-type">>, longstr, <<"quorum">>}, + {<<"x-expires">>, long, 1000}]})), + ok. + +consumer_priorities(Config) -> + Server = rabbit_ct_broker_helpers:get_node_config(Config, 0, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + qos(Ch, 2, false), + QQ = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', QQ, 0, 0}, + declare(Ch, QQ, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), + + %% consumer with default priority + Tag1 = <<"ctag1">>, + amqp_channel:subscribe(Ch, #'basic.consume'{queue = QQ, + no_ack = false, + consumer_tag = Tag1}, + self()), + receive + #'basic.consume_ok'{consumer_tag = Tag1} -> + ok + end, + %% consumer with higher priority + Tag2 = <<"ctag2">>, + amqp_channel:subscribe(Ch, #'basic.consume'{queue = QQ, + arguments = [{"x-priority", long, 10}], + no_ack = false, + consumer_tag = Tag2}, + self()), + receive + #'basic.consume_ok'{consumer_tag = Tag2} -> + ok + end, + + publish(Ch, QQ), + %% Tag2 should receive the message + DT1 = receive + {#'basic.deliver'{delivery_tag = D1, + consumer_tag = Tag2}, _} -> + D1 + after 5000 -> + flush(100), + ct:fail("basic.deliver timeout") + end, + publish(Ch, QQ), + %% Tag2 should receive the message + receive + {#'basic.deliver'{delivery_tag = _, + consumer_tag = Tag2}, _} -> + ok + after 5000 -> + flush(100), + ct:fail("basic.deliver timeout") + end, + + publish(Ch, QQ), + %% Tag1 should receive the message as Tag2 has maxed qos + receive + {#'basic.deliver'{delivery_tag = _, + consumer_tag = Tag1}, _} -> + ok + after 5000 -> + flush(100), + ct:fail("basic.deliver timeout") + end, + + ok = amqp_channel:cast(Ch, #'basic.ack'{delivery_tag = DT1, + multiple = false}), + publish(Ch, QQ), + %% Tag2 should receive the message + receive + {#'basic.deliver'{delivery_tag = _, + consumer_tag = Tag2}, _} -> + ok + after 5000 -> + flush(100), + ct:fail("basic.deliver timeout") + end, + + ok. + +%%---------------------------------------------------------------------------- + +declare(Ch, Q) -> + declare(Ch, Q, []). + +declare(Ch, Q, Args) -> + amqp_channel:call(Ch, #'queue.declare'{queue = Q, + durable = true, + auto_delete = false, + arguments = Args}). + +assert_queue_type(Server, Q, Expected) -> + Actual = get_queue_type(Server, Q), + Expected = Actual. + +get_queue_type(Server, Q0) -> + QNameRes = rabbit_misc:r(<<"/">>, queue, Q0), + {ok, Q1} = rpc:call(Server, rabbit_amqqueue, lookup, [QNameRes]), + amqqueue:get_type(Q1). + +publish_many(Ch, Queue, Count) -> + [publish(Ch, Queue) || _ <- lists:seq(1, Count)]. + +publish(Ch, Queue) -> + publish(Ch, Queue, <<"msg">>). + +publish(Ch, Queue, Msg) -> + ok = amqp_channel:cast(Ch, + #'basic.publish'{routing_key = Queue}, + #amqp_msg{props = #'P_basic'{delivery_mode = 2}, + payload = Msg}). + +consume(Ch, Queue, NoAck) -> + {GetOk, _} = Reply = amqp_channel:call(Ch, #'basic.get'{queue = Queue, + no_ack = NoAck}), + ?assertMatch({#'basic.get_ok'{}, #amqp_msg{payload = <<"msg">>}}, Reply), + GetOk#'basic.get_ok'.delivery_tag. + +consume_empty(Ch, Queue, NoAck) -> + ?assertMatch(#'basic.get_empty'{}, + amqp_channel:call(Ch, #'basic.get'{queue = Queue, + no_ack = NoAck})). + +subscribe(Ch, Queue, NoAck) -> + amqp_channel:subscribe(Ch, #'basic.consume'{queue = Queue, + no_ack = NoAck, + consumer_tag = <<"ctag">>}, + self()), + receive + #'basic.consume_ok'{consumer_tag = <<"ctag">>} -> + ok + end. + +qos(Ch, Prefetch, Global) -> + ?assertMatch(#'basic.qos_ok'{}, + amqp_channel:call(Ch, #'basic.qos'{global = Global, + prefetch_count = Prefetch})). + +receive_basic_deliver(Redelivered) -> + receive + {#'basic.deliver'{redelivered = R}, _} when R == Redelivered -> + ok + end. + +wait_for_cleanup(Server, Channel, Number) -> + wait_for_cleanup(Server, Channel, Number, 60). + +wait_for_cleanup(Server, Channel, Number, 0) -> + ?assertEqual(length(rpc:call(Server, rabbit_channel, list_queue_states, [Channel])), + Number); +wait_for_cleanup(Server, Channel, Number, N) -> + case length(rpc:call(Server, rabbit_channel, list_queue_states, [Channel])) of + Length when Number == Length -> + ok; + _ -> + timer:sleep(500), + wait_for_cleanup(Server, Channel, Number, N - 1) + end. + +wait_until(Condition) -> + wait_until(Condition, 60). + +wait_until(Condition, 0) -> + ?assertEqual(true, Condition()); +wait_until(Condition, N) -> + case Condition() of + true -> + ok; + _ -> + timer:sleep(500), + wait_until(Condition, N - 1) + end. + + +force_leader_change([Server | _] = Servers, Q) -> + RaName = ra_name(Q), + {ok, _, {_, Leader}} = ra:members({RaName, Server}), + [F1, _] = Servers -- [Leader], + ok = rpc:call(F1, ra, trigger_election, [{RaName, F1}]), + case ra:members({RaName, Leader}) of + {ok, _, {_, Leader}} -> + %% Leader has been re-elected + force_leader_change(Servers, Q); + {ok, _, _} -> + %% Leader has changed + ok + end. + +delete_queues() -> + [rabbit_amqqueue:delete(Q, false, false, <<"dummy">>) + || Q <- rabbit_amqqueue:list()]. + +stop_node(Config, Server) -> + rabbit_ct_broker_helpers:rabbitmqctl(Config, Server, ["stop"]). + +get_message_bytes(Leader, QRes) -> + case rpc:call(Leader, ets, lookup, [queue_metrics, QRes]) of + [{QRes, Props, _}] -> + {proplists:get_value(message_bytes, Props), + proplists:get_value(message_bytes_ready, Props), + proplists:get_value(message_bytes_unacknowledged, Props)}; + _ -> + [] + end. + +wait_for_consensus(Name, Config) -> + Server = rabbit_ct_broker_helpers:get_node_config(Config, 0, nodename), + RaName = ra_name(Name), + {ok, _, _} = ra:members({RaName, Server}). + +queue_names(Records) -> + [begin + #resource{name = Name} = amqqueue:get_name(Q), + Name + end || Q <- Records]. diff --git a/deps/rabbit/test/quorum_queue_utils.erl b/deps/rabbit/test/quorum_queue_utils.erl new file mode 100644 index 0000000000..224abeeeeb --- /dev/null +++ b/deps/rabbit/test/quorum_queue_utils.erl @@ -0,0 +1,112 @@ +-module(quorum_queue_utils). + +-include_lib("eunit/include/eunit.hrl"). + +-export([ + wait_for_messages_ready/3, + wait_for_messages_pending_ack/3, + wait_for_messages_total/3, + wait_for_messages/2, + dirty_query/3, + ra_name/1, + fifo_machines_use_same_version/1, + fifo_machines_use_same_version/2, + is_mixed_versions/0 + ]). + +wait_for_messages_ready(Servers, QName, Ready) -> + wait_for_messages(Servers, QName, Ready, + fun rabbit_fifo:query_messages_ready/1, 60). + +wait_for_messages_pending_ack(Servers, QName, Ready) -> + wait_for_messages(Servers, QName, Ready, + fun rabbit_fifo:query_messages_checked_out/1, 60). + +wait_for_messages_total(Servers, QName, Total) -> + wait_for_messages(Servers, QName, Total, + fun rabbit_fifo:query_messages_total/1, 60). + +wait_for_messages(Servers, QName, Number, Fun, 0) -> + Msgs = dirty_query(Servers, QName, Fun), + ?assertEqual([Number || _ <- lists:seq(1, length(Servers))], Msgs); +wait_for_messages(Servers, QName, Number, Fun, N) -> + Msgs = dirty_query(Servers, QName, Fun), + ct:pal("Got messages ~p ~p", [QName, Msgs]), + %% hack to allow the check to succeed in mixed versions clusters if at + %% least one node matches the criteria rather than all nodes for + F = case is_mixed_versions() of + true -> + any; + false -> + all + end, + case lists:F(fun(C) when is_integer(C) -> + C == Number; + (_) -> + false + end, Msgs) of + true -> + ok; + _ -> + timer:sleep(500), + wait_for_messages(Servers, QName, Number, Fun, N - 1) + end. + +wait_for_messages(Config, Stats) -> + wait_for_messages(Config, lists:sort(Stats), 60). + +wait_for_messages(Config, Stats, 0) -> + ?assertEqual(Stats, + lists:sort( + filter_queues(Stats, + rabbit_ct_broker_helpers:rabbitmqctl_list( + Config, 0, ["list_queues", "name", "messages", "messages_ready", + "messages_unacknowledged"])))); +wait_for_messages(Config, Stats, N) -> + case lists:sort( + filter_queues(Stats, + rabbit_ct_broker_helpers:rabbitmqctl_list( + Config, 0, ["list_queues", "name", "messages", "messages_ready", + "messages_unacknowledged"]))) of + Stats0 when Stats0 == Stats -> + ok; + _ -> + timer:sleep(500), + wait_for_messages(Config, Stats, N - 1) + end. + +dirty_query(Servers, QName, Fun) -> + lists:map( + fun(N) -> + case rpc:call(N, ra, local_query, [{QName, N}, Fun]) of + {ok, {_, Msgs}, _} -> + Msgs; + _E -> + undefined + end + end, Servers). + +ra_name(Q) -> + binary_to_atom(<<"%2F_", Q/binary>>, utf8). + +filter_queues(Expected, Got) -> + Keys = [K || [K, _, _, _] <- Expected], + lists:filter(fun([K, _, _, _]) -> + lists:member(K, Keys) + end, Got). + +fifo_machines_use_same_version(Config) -> + Nodenames = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + fifo_machines_use_same_version(Config, Nodenames). + +fifo_machines_use_same_version(Config, Nodenames) + when length(Nodenames) >= 1 -> + [MachineAVersion | OtherMachinesVersions] = + [(catch rabbit_ct_broker_helpers:rpc( + Config, Nodename, + rabbit_fifo, version, [])) + || Nodename <- Nodenames], + lists:all(fun(V) -> V =:= MachineAVersion end, OtherMachinesVersions). + +is_mixed_versions() -> + not (false == os:getenv("SECONDARY_UMBRELLA")). diff --git a/deps/rabbit/test/rabbit_auth_backend_context_propagation_mock.erl b/deps/rabbit/test/rabbit_auth_backend_context_propagation_mock.erl new file mode 100644 index 0000000000..e721f5e0dd --- /dev/null +++ b/deps/rabbit/test/rabbit_auth_backend_context_propagation_mock.erl @@ -0,0 +1,46 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2019-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +%% A mock authn/authz that records information during calls. For testing purposes only. + +-module(rabbit_auth_backend_context_propagation_mock). +-include_lib("rabbit_common/include/rabbit.hrl"). + +-behaviour(rabbit_authn_backend). +-behaviour(rabbit_authz_backend). + +-export([user_login_authentication/2, user_login_authorization/2, + check_vhost_access/3, check_resource_access/4, check_topic_access/4, + state_can_expire/0, + get/1, init/0]). + +init() -> + ets:new(?MODULE, [set, public, named_table]). + +user_login_authentication(_, AuthProps) -> + ets:insert(?MODULE, {authentication, AuthProps}), + {ok, #auth_user{username = <<"dummy">>, + tags = [], + impl = none}}. + +user_login_authorization(_, _) -> + {ok, does_not_matter}. + +check_vhost_access(#auth_user{}, _VHostPath, AuthzData) -> + ets:insert(?MODULE, {vhost_access, AuthzData}), + true. +check_resource_access(#auth_user{}, #resource{}, _Permission, AuthzContext) -> + ets:insert(?MODULE, {resource_access, AuthzContext}), + true. +check_topic_access(#auth_user{}, #resource{}, _Permission, TopicContext) -> + ets:insert(?MODULE, {topic_access, TopicContext}), + true. + +state_can_expire() -> false. + +get(K) -> + ets:lookup(?MODULE, K). diff --git a/deps/rabbit/test/rabbit_confirms_SUITE.erl b/deps/rabbit/test/rabbit_confirms_SUITE.erl new file mode 100644 index 0000000000..331c3ca7c3 --- /dev/null +++ b/deps/rabbit/test/rabbit_confirms_SUITE.erl @@ -0,0 +1,154 @@ +-module(rabbit_confirms_SUITE). + +-compile(export_all). + +-export([ + ]). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +%%%=================================================================== +%%% Common Test callbacks +%%%=================================================================== + +all() -> + [ + {group, tests} + ]. + + +all_tests() -> + [ + confirm, + reject, + remove_queue + ]. + +groups() -> + [ + {tests, [], all_tests()} + ]. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_Group, Config) -> + Config. + +end_per_group(_Group, _Config) -> + ok. + +init_per_testcase(_TestCase, Config) -> + Config. + +end_per_testcase(_TestCase, _Config) -> + ok. + +%%%=================================================================== +%%% Test cases +%%%=================================================================== + +confirm(_Config) -> + XName = rabbit_misc:r(<<"/">>, exchange, <<"X">>), + QName = rabbit_misc:r(<<"/">>, queue, <<"Q">>), + QName2 = rabbit_misc:r(<<"/">>, queue, <<"Q2">>), + U0 = rabbit_confirms:init(), + ?assertEqual(0, rabbit_confirms:size(U0)), + ?assertEqual(undefined, rabbit_confirms:smallest(U0)), + ?assertEqual(true, rabbit_confirms:is_empty(U0)), + + U1 = rabbit_confirms:insert(1, [QName], XName, U0), + ?assertEqual(1, rabbit_confirms:size(U1)), + ?assertEqual(1, rabbit_confirms:smallest(U1)), + ?assertEqual(false, rabbit_confirms:is_empty(U1)), + + {[{1, XName}], U2} = rabbit_confirms:confirm([1], QName, U1), + ?assertEqual(0, rabbit_confirms:size(U2)), + ?assertEqual(undefined, rabbit_confirms:smallest(U2)), + ?assertEqual(true, rabbit_confirms:is_empty(U2)), + + U3 = rabbit_confirms:insert(2, [QName], XName, U1), + ?assertEqual(2, rabbit_confirms:size(U3)), + ?assertEqual(1, rabbit_confirms:smallest(U3)), + ?assertEqual(false, rabbit_confirms:is_empty(U3)), + + {[{1, XName}], U4} = rabbit_confirms:confirm([1], QName, U3), + ?assertEqual(1, rabbit_confirms:size(U4)), + ?assertEqual(2, rabbit_confirms:smallest(U4)), + ?assertEqual(false, rabbit_confirms:is_empty(U4)), + + U5 = rabbit_confirms:insert(2, [QName, QName2], XName, U1), + ?assertEqual(2, rabbit_confirms:size(U5)), + ?assertEqual(1, rabbit_confirms:smallest(U5)), + ?assertEqual(false, rabbit_confirms:is_empty(U5)), + + {[{1, XName}], U6} = rabbit_confirms:confirm([1, 2], QName, U5), + ?assertEqual(2, rabbit_confirms:smallest(U6)), + + {[{2, XName}], U7} = rabbit_confirms:confirm([2], QName2, U6), + ?assertEqual(0, rabbit_confirms:size(U7)), + ?assertEqual(undefined, rabbit_confirms:smallest(U7)), + + + U8 = rabbit_confirms:insert(2, [QName], XName, U1), + {[{1, XName}, {2, XName}], _U9} = rabbit_confirms:confirm([1, 2], QName, U8), + ok. + + +reject(_Config) -> + XName = rabbit_misc:r(<<"/">>, exchange, <<"X">>), + QName = rabbit_misc:r(<<"/">>, queue, <<"Q">>), + QName2 = rabbit_misc:r(<<"/">>, queue, <<"Q2">>), + U0 = rabbit_confirms:init(), + ?assertEqual(0, rabbit_confirms:size(U0)), + ?assertEqual(undefined, rabbit_confirms:smallest(U0)), + ?assertEqual(true, rabbit_confirms:is_empty(U0)), + + U1 = rabbit_confirms:insert(1, [QName], XName, U0), + + {ok, {1, XName}, U2} = rabbit_confirms:reject(1, U1), + {error, not_found} = rabbit_confirms:reject(1, U2), + ?assertEqual(0, rabbit_confirms:size(U2)), + ?assertEqual(undefined, rabbit_confirms:smallest(U2)), + + U3 = rabbit_confirms:insert(2, [QName, QName2], XName, U1), + + {ok, {1, XName}, U4} = rabbit_confirms:reject(1, U3), + {error, not_found} = rabbit_confirms:reject(1, U4), + ?assertEqual(1, rabbit_confirms:size(U4)), + ?assertEqual(2, rabbit_confirms:smallest(U4)), + + {ok, {2, XName}, U5} = rabbit_confirms:reject(2, U3), + {error, not_found} = rabbit_confirms:reject(2, U5), + ?assertEqual(1, rabbit_confirms:size(U5)), + ?assertEqual(1, rabbit_confirms:smallest(U5)), + + ok. + +remove_queue(_Config) -> + XName = rabbit_misc:r(<<"/">>, exchange, <<"X">>), + QName = rabbit_misc:r(<<"/">>, queue, <<"Q">>), + QName2 = rabbit_misc:r(<<"/">>, queue, <<"Q2">>), + U0 = rabbit_confirms:init(), + + U1 = rabbit_confirms:insert(1, [QName, QName2], XName, U0), + U2 = rabbit_confirms:insert(2, [QName2], XName, U1), + {[{2, XName}], U3} = rabbit_confirms:remove_queue(QName2, U2), + ?assertEqual(1, rabbit_confirms:size(U3)), + ?assertEqual(1, rabbit_confirms:smallest(U3)), + {[{1, XName}], U4} = rabbit_confirms:remove_queue(QName, U3), + ?assertEqual(0, rabbit_confirms:size(U4)), + ?assertEqual(undefined, rabbit_confirms:smallest(U4)), + + U5 = rabbit_confirms:insert(1, [QName], XName, U0), + U6 = rabbit_confirms:insert(2, [QName], XName, U5), + {[{1, XName}, {2, XName}], _U} = rabbit_confirms:remove_queue(QName, U6), + + ok. + + +%% Utility diff --git a/deps/rabbit/test/rabbit_core_metrics_gc_SUITE.erl b/deps/rabbit/test/rabbit_core_metrics_gc_SUITE.erl new file mode 100644 index 0000000000..cae5502a0a --- /dev/null +++ b/deps/rabbit/test/rabbit_core_metrics_gc_SUITE.erl @@ -0,0 +1,392 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(rabbit_core_metrics_gc_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). + +-compile(export_all). + +all() -> + [ + {group, non_parallel_tests}, + {group, cluster_tests} + ]. + +groups() -> + [ + {non_parallel_tests, [], + [ queue_metrics, + connection_metrics, + channel_metrics, + node_metrics, + gen_server2_metrics, + consumer_metrics + ] + }, + {cluster_tests, [], [cluster_queue_metrics]} + ]. + +%% ------------------------------------------------------------------- +%% Testsuite setup/teardown. +%% ------------------------------------------------------------------- + +merge_app_env(Config) -> + AppEnv = {rabbit, [{core_metrics_gc_interval, 6000000}, + {collect_statistics_interval, 100}, + {collect_statistics, fine}]}, + rabbit_ct_helpers:merge_app_env(Config, AppEnv). + +init_per_group(cluster_tests, Config) -> + rabbit_ct_helpers:log_environment(), + Conf = [{rmq_nodename_suffix, cluster_tests}, {rmq_nodes_count, 2}], + Config1 = rabbit_ct_helpers:set_config(Config, Conf), + rabbit_ct_helpers:run_setup_steps(Config1, setup_steps()); +init_per_group(non_parallel_tests, Config) -> + rabbit_ct_helpers:log_environment(), + Conf = [{rmq_nodename_suffix, non_parallel_tests}], + Config1 = rabbit_ct_helpers:set_config(Config, Conf), + rabbit_ct_helpers:run_setup_steps(Config1, setup_steps()). + +end_per_group(_, Config) -> + rabbit_ct_helpers:run_teardown_steps( + Config, + rabbit_ct_broker_helpers:teardown_steps()). + +init_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_started(Config, Testcase), + rabbit_ct_helpers:run_steps(Config, + rabbit_ct_client_helpers:setup_steps()). + +end_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_finished(Config, Testcase), + rabbit_ct_helpers:run_teardown_steps( + Config, + rabbit_ct_client_helpers:teardown_steps()). + +setup_steps() -> + [ fun merge_app_env/1 ] ++ rabbit_ct_broker_helpers:setup_steps(). + +%% ------------------------------------------------------------------- +%% Single-node Testcases. +%% ------------------------------------------------------------------- + +queue_metrics(Config) -> + A = rabbit_ct_broker_helpers:get_node_config(Config, 0, nodename), + Ch = rabbit_ct_client_helpers:open_channel(Config, A), + + amqp_channel:call(Ch, #'queue.declare'{queue = <<"queue_metrics">>}), + amqp_channel:cast(Ch, #'basic.publish'{routing_key = <<"queue_metrics">>}, + #amqp_msg{payload = <<"hello">>}), + timer:sleep(150), + + Q = q(<<"myqueue">>), + + rabbit_ct_broker_helpers:rpc(Config, A, rabbit_core_metrics, queue_stats, + [Q, infos]), + rabbit_ct_broker_helpers:rpc(Config, A, rabbit_core_metrics, queue_stats, + [Q, 1, 1, 1, 1]), + + [_] = rabbit_ct_broker_helpers:rpc(Config, A, ets, lookup, + [queue_metrics, Q]), + [_] = rabbit_ct_broker_helpers:rpc(Config, A, ets, lookup, + [queue_coarse_metrics, Q]), + %% Trigger gc. When the gen_server:call returns, the gc has already finished. + rabbit_ct_broker_helpers:rpc(Config, A, erlang, send, [rabbit_core_metrics_gc, start_gc]), + rabbit_ct_broker_helpers:rpc(Config, A, gen_server, call, [rabbit_core_metrics_gc, test]), + + [_|_] = rabbit_ct_broker_helpers:rpc(Config, A, ets, tab2list, + [queue_metrics]), + [_|_] = rabbit_ct_broker_helpers:rpc(Config, A, ets, tab2list, + [queue_coarse_metrics]), + + [] = rabbit_ct_broker_helpers:rpc(Config, A, ets, lookup, + [queue_metrics, Q]), + [] = rabbit_ct_broker_helpers:rpc(Config, A, ets, lookup, + [queue_coarse_metrics, Q]), + + amqp_channel:call(Ch, #'queue.delete'{queue = <<"queue_metrics">>}), + rabbit_ct_client_helpers:close_channel(Ch), + + ok. + +connection_metrics(Config) -> + A = rabbit_ct_broker_helpers:get_node_config(Config, 0, nodename), + Ch = rabbit_ct_client_helpers:open_channel(Config, A), + + amqp_channel:call(Ch, #'queue.declare'{queue = <<"queue_metrics">>}), + amqp_channel:cast(Ch, #'basic.publish'{routing_key = <<"queue_metrics">>}, + #amqp_msg{payload = <<"hello">>}), + timer:sleep(200), + + DeadPid = rabbit_ct_broker_helpers:rpc(Config, A, ?MODULE, dead_pid, []), + + rabbit_ct_broker_helpers:rpc(Config, A, rabbit_core_metrics, + connection_created, [DeadPid, infos]), + rabbit_ct_broker_helpers:rpc(Config, A, rabbit_core_metrics, + connection_stats, [DeadPid, infos]), + rabbit_ct_broker_helpers:rpc(Config, A, rabbit_core_metrics, + connection_stats, [DeadPid, 1, 1, 1]), + + [_] = rabbit_ct_broker_helpers:rpc(Config, A, ets, lookup, + [connection_created, DeadPid]), + [_] = rabbit_ct_broker_helpers:rpc(Config, A, ets, lookup, + [connection_metrics, DeadPid]), + [_] = rabbit_ct_broker_helpers:rpc(Config, A, ets, lookup, + [connection_coarse_metrics, DeadPid]), + + %% Trigger gc. When the gen_server:call returns, the gc has already finished. + rabbit_ct_broker_helpers:rpc(Config, A, erlang, send, [rabbit_core_metrics_gc, start_gc]), + rabbit_ct_broker_helpers:rpc(Config, A, gen_server, call, [rabbit_core_metrics_gc, test]), + + [] = rabbit_ct_broker_helpers:rpc(Config, A, ets, lookup, + [connection_created, DeadPid]), + [] = rabbit_ct_broker_helpers:rpc(Config, A, ets, lookup, + [connection_metrics, DeadPid]), + [] = rabbit_ct_broker_helpers:rpc(Config, A, ets, lookup, + [connection_coarse_metrics, DeadPid]), + + [_|_] = rabbit_ct_broker_helpers:rpc(Config, A, ets, tab2list, [connection_created]), + [_|_] = rabbit_ct_broker_helpers:rpc(Config, A, ets, tab2list, [connection_metrics]), + [_|_] = rabbit_ct_broker_helpers:rpc(Config, A, ets, tab2list, [connection_coarse_metrics]), + + amqp_channel:call(Ch, #'queue.delete'{queue = <<"queue_metrics">>}), + rabbit_ct_client_helpers:close_channel(Ch), + + ok. + +channel_metrics(Config) -> + A = rabbit_ct_broker_helpers:get_node_config(Config, 0, nodename), + Ch = rabbit_ct_client_helpers:open_channel(Config, A), + + amqp_channel:call(Ch, #'queue.declare'{queue = <<"queue_metrics">>}), + amqp_channel:cast(Ch, #'basic.publish'{routing_key = <<"queue_metrics">>}, + #amqp_msg{payload = <<"hello">>}), + amqp_channel:cast(Ch, #'basic.publish'{routing_key = <<"won't route $¢% anywhere">>}, + #amqp_msg{payload = <<"hello">>}), + {#'basic.get_ok'{}, _} = amqp_channel:call(Ch, #'basic.get'{queue = <<"queue_metrics">>, + no_ack=true}), + timer:sleep(150), + + DeadPid = rabbit_ct_broker_helpers:rpc(Config, A, ?MODULE, dead_pid, []), + + Q = q(<<"myqueue">>), + X = x(<<"myexchange">>), + + + rabbit_ct_broker_helpers:rpc(Config, A, rabbit_core_metrics, + channel_created, [DeadPid, infos]), + rabbit_ct_broker_helpers:rpc(Config, A, rabbit_core_metrics, + channel_stats, [DeadPid, infos]), + rabbit_ct_broker_helpers:rpc(Config, A, rabbit_core_metrics, + channel_stats, [reductions, DeadPid, 1]), + rabbit_ct_broker_helpers:rpc(Config, A, rabbit_core_metrics, + channel_stats, [exchange_stats, publish, + {DeadPid, X}, 1]), + rabbit_ct_broker_helpers:rpc(Config, A, rabbit_core_metrics, + channel_stats, [queue_stats, get, + {DeadPid, Q}, 1]), + rabbit_ct_broker_helpers:rpc(Config, A, rabbit_core_metrics, + channel_stats, [queue_exchange_stats, publish, + {DeadPid, {Q, X}}, 1]), + + [_] = rabbit_ct_broker_helpers:rpc(Config, A, ets, lookup, + [channel_created, DeadPid]), + [_] = rabbit_ct_broker_helpers:rpc(Config, A, ets, lookup, + [channel_metrics, DeadPid]), + [_] = rabbit_ct_broker_helpers:rpc(Config, A, ets, lookup, + [channel_process_metrics, DeadPid]), + [_] = rabbit_ct_broker_helpers:rpc(Config, A, ets, lookup, + [channel_exchange_metrics, {DeadPid, X}]), + [_] = rabbit_ct_broker_helpers:rpc(Config, A, ets, lookup, + [channel_queue_metrics, {DeadPid, Q}]), + [_] = rabbit_ct_broker_helpers:rpc(Config, A, ets, lookup, + [channel_queue_exchange_metrics, {DeadPid, {Q, X}}]), + + %% Trigger gc. When the gen_server:call returns, the gc has already finished. + rabbit_ct_broker_helpers:rpc(Config, A, erlang, send, [rabbit_core_metrics_gc, start_gc]), + rabbit_ct_broker_helpers:rpc(Config, A, gen_server, call, [rabbit_core_metrics_gc, test]), + + + [_|_] = rabbit_ct_broker_helpers:rpc(Config, A, ets, tab2list, [channel_created]), + [_|_] = rabbit_ct_broker_helpers:rpc(Config, A, ets, tab2list, [channel_metrics]), + [_|_] = rabbit_ct_broker_helpers:rpc(Config, A, ets, tab2list, [channel_process_metrics]), + [_|_] = rabbit_ct_broker_helpers:rpc(Config, A, ets, tab2list, [channel_exchange_metrics]), + [_|_] = rabbit_ct_broker_helpers:rpc(Config, A, ets, tab2list, [channel_queue_metrics]), + [_|_] = rabbit_ct_broker_helpers:rpc(Config, A, ets, tab2list, [channel_queue_exchange_metrics]), + + [] = rabbit_ct_broker_helpers:rpc(Config, A, ets, lookup, + [channel_created, DeadPid]), + [] = rabbit_ct_broker_helpers:rpc(Config, A, ets, lookup, + [channel_metrics, DeadPid]), + [] = rabbit_ct_broker_helpers:rpc(Config, A, ets, lookup, + [channel_process_metrics, DeadPid]), + [] = rabbit_ct_broker_helpers:rpc(Config, A, ets, lookup, + [channel_exchange_metrics, {DeadPid, X}]), + [] = rabbit_ct_broker_helpers:rpc(Config, A, ets, lookup, + [channel_queue_metrics, {DeadPid, Q}]), + [] = rabbit_ct_broker_helpers:rpc(Config, A, ets, lookup, + [channel_queue_exchange_metrics, {DeadPid, {Q, X}}]), + + amqp_channel:call(Ch, #'queue.delete'{queue = <<"queue_metrics">>}), + rabbit_ct_client_helpers:close_channel(Ch), + + ok. + +node_metrics(Config) -> + A = rabbit_ct_broker_helpers:get_node_config(Config, 0, nodename), + + rabbit_ct_broker_helpers:rpc(Config, A, rabbit_core_metrics, node_node_stats, + [{node(), 'deer@localhost'}, infos]), + + [_] = rabbit_ct_broker_helpers:rpc(Config, A, ets, lookup, + [node_node_metrics, {node(), 'deer@localhost'}]), + + %% Trigger gc. When the gen_server:call returns, the gc has already finished. + rabbit_ct_broker_helpers:rpc(Config, A, erlang, send, [rabbit_core_metrics_gc, start_gc]), + rabbit_ct_broker_helpers:rpc(Config, A, gen_server, call, [rabbit_core_metrics_gc, test]), + + [] = rabbit_ct_broker_helpers:rpc(Config, A, ets, lookup, + [node_node_metrics, {node(), 'deer@localhost'}]), + + ok. + +gen_server2_metrics(Config) -> + A = rabbit_ct_broker_helpers:get_node_config(Config, 0, nodename), + + DeadPid = rabbit_ct_broker_helpers:rpc(Config, A, ?MODULE, dead_pid, []), + + rabbit_ct_broker_helpers:rpc(Config, A, rabbit_core_metrics, gen_server2_stats, + [DeadPid, 1]), + + [_] = rabbit_ct_broker_helpers:rpc(Config, A, ets, lookup, + [gen_server2_metrics, DeadPid]), + + %% Trigger gc. When the gen_server:call returns, the gc has already finished. + rabbit_ct_broker_helpers:rpc(Config, A, erlang, send, [rabbit_core_metrics_gc, start_gc]), + rabbit_ct_broker_helpers:rpc(Config, A, gen_server, call, [rabbit_core_metrics_gc, test]), + + [_|_] = rabbit_ct_broker_helpers:rpc(Config, A, ets, tab2list, [gen_server2_metrics]), + + [] = rabbit_ct_broker_helpers:rpc(Config, A, ets, lookup, + [gen_server2_metrics, DeadPid]), + + ok. + +consumer_metrics(Config) -> + A = rabbit_ct_broker_helpers:get_node_config(Config, 0, nodename), + Ch = rabbit_ct_client_helpers:open_channel(Config, A), + + amqp_channel:call(Ch, #'queue.declare'{queue = <<"queue_metrics">>}), + amqp_channel:call(Ch, #'basic.consume'{queue = <<"queue_metrics">>}), + timer:sleep(200), + + DeadPid = rabbit_ct_broker_helpers:rpc(Config, A, ?MODULE, dead_pid, []), + + QName = q(<<"queue_metrics">>), + CTag = <<"tag">>, + rabbit_ct_broker_helpers:rpc(Config, A, rabbit_core_metrics, + consumer_created, [DeadPid, CTag, true, true, + QName, 1, false, waiting, []]), + Id = {QName, DeadPid, CTag}, + [_] = rabbit_ct_broker_helpers:rpc(Config, A, ets, lookup, [consumer_created, Id]), + + %% Trigger gc. When the gen_server:call returns, the gc has already finished. + rabbit_ct_broker_helpers:rpc(Config, A, erlang, send, [rabbit_core_metrics_gc, start_gc]), + rabbit_ct_broker_helpers:rpc(Config, A, gen_server, call, [rabbit_core_metrics_gc, test]), + + [_|_] = rabbit_ct_broker_helpers:rpc(Config, A, ets, tab2list, [consumer_created]), + [] = rabbit_ct_broker_helpers:rpc(Config, A, ets, lookup, [consumer_created, Id]), + + amqp_channel:call(Ch, #'queue.delete'{queue = <<"queue_metrics">>}), + rabbit_ct_client_helpers:close_channel(Ch), + + ok. + +dead_pid() -> + spawn(fun() -> ok end). + +q(Name) -> + #resource{ virtual_host = <<"/">>, + kind = queue, + name = Name }. + +x(Name) -> + #resource{ virtual_host = <<"/">>, + kind = exchange, + name = Name }. + +%% ------------------------------------------------------------------- +%% Cluster Testcases. +%% ------------------------------------------------------------------- + +cluster_queue_metrics(Config) -> + VHost = <<"/">>, + QueueName = <<"cluster_queue_metrics">>, + PolicyName = <<"ha-policy-1">>, + PolicyPattern = <<".*">>, + PolicyAppliesTo = <<"queues">>, + + Node0 = rabbit_ct_broker_helpers:get_node_config(Config, 0, nodename), + Node1 = rabbit_ct_broker_helpers:get_node_config(Config, 1, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Node0), + + Node0Name = rabbit_data_coercion:to_binary(Node0), + Definition0 = [{<<"ha-mode">>, <<"nodes">>}, {<<"ha-params">>, [Node0Name]}], + ok = rabbit_ct_broker_helpers:set_policy(Config, 0, + PolicyName, PolicyPattern, + PolicyAppliesTo, Definition0), + + amqp_channel:call(Ch, #'queue.declare'{queue = QueueName}), + amqp_channel:call(Ch, #'basic.publish'{routing_key = QueueName}, + #amqp_msg{payload = <<"hello">>}), + + % Update policy to point to other node + Node1Name = rabbit_data_coercion:to_binary(Node1), + Definition1 = [{<<"ha-mode">>, <<"nodes">>}, {<<"ha-params">>, [Node1Name]}], + ok = rabbit_ct_broker_helpers:set_policy(Config, 0, + PolicyName, PolicyPattern, + PolicyAppliesTo, Definition1), + + % Synchronize + Name = rabbit_misc:r(VHost, queue, QueueName), + [Q] = rabbit_ct_broker_helpers:rpc(Config, Node0, ets, lookup, [rabbit_queue, Name]), + QPid = amqqueue:get_pid(Q), + ok = rabbit_ct_broker_helpers:rpc(Config, Node0, rabbit_amqqueue, sync_mirrors, [QPid]), + + % Check ETS table for data + wait_for(fun () -> + [] =:= rabbit_ct_broker_helpers:rpc( + Config, Node0, ets, tab2list, + [queue_coarse_metrics]) + end, 60), + + wait_for(fun () -> + Ret = rabbit_ct_broker_helpers:rpc( + Config, Node1, ets, tab2list, + [queue_coarse_metrics]), + case Ret of + [{Name, 1, 0, 1, _}] -> true; + _ -> false + end + end, 60), + + amqp_channel:call(Ch, #'queue.delete'{queue=QueueName}), + rabbit_ct_client_helpers:close_channel(Ch), + Config. + +wait_for(_Fun, 0) -> false; +wait_for(Fun, Seconds) -> + case Fun() of + true -> ok; + false -> + timer:sleep(1000), + wait_for(Fun, Seconds - 1) + end. diff --git a/deps/rabbit/test/rabbit_dummy_protocol_connection_info.erl b/deps/rabbit/test/rabbit_dummy_protocol_connection_info.erl new file mode 100644 index 0000000000..92c01d2b0e --- /dev/null +++ b/deps/rabbit/test/rabbit_dummy_protocol_connection_info.erl @@ -0,0 +1,19 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2017-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +%% Dummy module to test rabbit_direct:extract_extra_auth_props + +-module(rabbit_dummy_protocol_connection_info). + +%% API +-export([additional_authn_params/4]). + +additional_authn_params(_Creds, _VHost, Pid, _Infos) -> + case Pid of + -1 -> throw(error); + _ -> [{client_id, <<"DummyClientId">>}] + end. diff --git a/deps/rabbit/test/rabbit_fifo_SUITE.erl b/deps/rabbit/test/rabbit_fifo_SUITE.erl new file mode 100644 index 0000000000..7b90d91bfa --- /dev/null +++ b/deps/rabbit/test/rabbit_fifo_SUITE.erl @@ -0,0 +1,1667 @@ +-module(rabbit_fifo_SUITE). + +%% rabbit_fifo unit tests suite + +-compile(export_all). + +-compile({no_auto_import, [apply/3]}). +-export([ + ]). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("rabbit_common/include/rabbit.hrl"). +-include("src/rabbit_fifo.hrl"). + +%%%=================================================================== +%%% Common Test callbacks +%%%=================================================================== + +all() -> + [ + {group, tests} + ]. + + +%% replicate eunit like test resultion +all_tests() -> + [F || {F, _} <- ?MODULE:module_info(functions), + re:run(atom_to_list(F), "_test$") /= nomatch]. + +groups() -> + [ + {tests, [], all_tests()} + ]. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_Group, Config) -> + Config. + +end_per_group(_Group, _Config) -> + ok. + +init_per_testcase(_TestCase, Config) -> + Config. + +end_per_testcase(_TestCase, _Config) -> + ok. + +%%%=================================================================== +%%% Test cases +%%%=================================================================== + +-define(ASSERT_EFF(EfxPat, Effects), + ?ASSERT_EFF(EfxPat, true, Effects)). + +-define(ASSERT_EFF(EfxPat, Guard, Effects), + ?assert(lists:any(fun (EfxPat) when Guard -> true; + (_) -> false + end, Effects))). + +-define(ASSERT_NO_EFF(EfxPat, Effects), + ?assert(not lists:any(fun (EfxPat) -> true; + (_) -> false + end, Effects))). + +-define(ASSERT_NO_EFF(EfxPat, Guard, Effects), + ?assert(not lists:any(fun (EfxPat) when Guard -> true; + (_) -> false + end, Effects))). + +-define(assertNoEffect(EfxPat, Effects), + ?assert(not lists:any(fun (EfxPat) -> true; + (_) -> false + end, Effects))). + +test_init(Name) -> + init(#{name => Name, + queue_resource => rabbit_misc:r("/", queue, + atom_to_binary(Name, utf8)), + release_cursor_interval => 0}). + +enq_enq_checkout_test(_) -> + Cid = {<<"enq_enq_checkout_test">>, self()}, + {State1, _} = enq(1, 1, first, test_init(test)), + {State2, _} = enq(2, 2, second, State1), + {_State3, _, Effects} = + apply(meta(3), + rabbit_fifo:make_checkout(Cid, {once, 2, simple_prefetch}, #{}), + State2), + ?ASSERT_EFF({monitor, _, _}, Effects), + ?ASSERT_EFF({send_msg, _, {delivery, _, _}, _}, Effects), + ok. + +credit_enq_enq_checkout_settled_credit_test(_) -> + Cid = {?FUNCTION_NAME, self()}, + {State1, _} = enq(1, 1, first, test_init(test)), + {State2, _} = enq(2, 2, second, State1), + {State3, _, Effects} = + apply(meta(3), rabbit_fifo:make_checkout(Cid, {auto, 1, credited}, #{}), State2), + ?ASSERT_EFF({monitor, _, _}, Effects), + Deliveries = lists:filter(fun ({send_msg, _, {delivery, _, _}, _}) -> true; + (_) -> false + end, Effects), + ?assertEqual(1, length(Deliveries)), + %% settle the delivery this should _not_ result in further messages being + %% delivered + {State4, SettledEffects} = settle(Cid, 4, 1, State3), + ?assertEqual(false, lists:any(fun ({send_msg, _, {delivery, _, _}, _}) -> + true; + (_) -> false + end, SettledEffects)), + %% granting credit (3) should deliver the second msg if the receivers + %% delivery count is (1) + {State5, CreditEffects} = credit(Cid, 5, 1, 1, false, State4), + % ?debugFmt("CreditEffects ~p ~n~p", [CreditEffects, State4]), + ?ASSERT_EFF({send_msg, _, {delivery, _, _}, _}, CreditEffects), + {_State6, FinalEffects} = enq(6, 3, third, State5), + ?assertEqual(false, lists:any(fun ({send_msg, _, {delivery, _, _}, _}) -> + true; + (_) -> false + end, FinalEffects)), + ok. + +credit_with_drained_test(_) -> + Cid = {?FUNCTION_NAME, self()}, + State0 = test_init(test), + %% checkout with a single credit + {State1, _, _} = + apply(meta(1), rabbit_fifo:make_checkout(Cid, {auto, 1, credited},#{}), + State0), + ?assertMatch(#rabbit_fifo{consumers = #{Cid := #consumer{credit = 1, + delivery_count = 0}}}, + State1), + {State, Result, _} = + apply(meta(3), rabbit_fifo:make_credit(Cid, 0, 5, true), State1), + ?assertMatch(#rabbit_fifo{consumers = #{Cid := #consumer{credit = 0, + delivery_count = 5}}}, + State), + ?assertEqual({multi, [{send_credit_reply, 0}, + {send_drained, {?FUNCTION_NAME, 5}}]}, + Result), + ok. + +credit_and_drain_test(_) -> + Cid = {?FUNCTION_NAME, self()}, + {State1, _} = enq(1, 1, first, test_init(test)), + {State2, _} = enq(2, 2, second, State1), + %% checkout without any initial credit (like AMQP 1.0 would) + {State3, _, CheckEffs} = + apply(meta(3), rabbit_fifo:make_checkout(Cid, {auto, 0, credited}, #{}), + State2), + + ?ASSERT_NO_EFF({send_msg, _, {delivery, _, _}}, CheckEffs), + {State4, {multi, [{send_credit_reply, 0}, + {send_drained, {?FUNCTION_NAME, 2}}]}, + Effects} = apply(meta(4), rabbit_fifo:make_credit(Cid, 4, 0, true), State3), + ?assertMatch(#rabbit_fifo{consumers = #{Cid := #consumer{credit = 0, + delivery_count = 4}}}, + State4), + + ?ASSERT_EFF({send_msg, _, {delivery, _, [{_, {_, first}}, + {_, {_, second}}]}, _}, Effects), + {_State5, EnqEffs} = enq(5, 2, third, State4), + ?ASSERT_NO_EFF({send_msg, _, {delivery, _, _}}, EnqEffs), + ok. + + + +enq_enq_deq_test(_) -> + Cid = {?FUNCTION_NAME, self()}, + {State1, _} = enq(1, 1, first, test_init(test)), + {State2, _} = enq(2, 2, second, State1), + % get returns a reply value + NumReady = 1, + {_State3, {dequeue, {0, {_, first}}, NumReady}, [{monitor, _, _}]} = + apply(meta(3), rabbit_fifo:make_checkout(Cid, {dequeue, unsettled}, #{}), + State2), + ok. + +enq_enq_deq_deq_settle_test(_) -> + Cid = {?FUNCTION_NAME, self()}, + {State1, _} = enq(1, 1, first, test_init(test)), + {State2, _} = enq(2, 2, second, State1), + % get returns a reply value + {State3, {dequeue, {0, {_, first}}, 1}, [{monitor, _, _}]} = + apply(meta(3), rabbit_fifo:make_checkout(Cid, {dequeue, unsettled}, #{}), + State2), + {_State4, {dequeue, empty}} = + apply(meta(4), rabbit_fifo:make_checkout(Cid, {dequeue, unsettled}, #{}), + State3), + ok. + +enq_enq_checkout_get_settled_test(_) -> + Cid = {?FUNCTION_NAME, self()}, + {State1, _} = enq(1, 1, first, test_init(test)), + % get returns a reply value + {_State2, {dequeue, {0, {_, first}}, _}, _Effs} = + apply(meta(3), rabbit_fifo:make_checkout(Cid, {dequeue, settled}, #{}), + State1), + ok. + +checkout_get_empty_test(_) -> + Cid = {?FUNCTION_NAME, self()}, + State = test_init(test), + {_State2, {dequeue, empty}} = + apply(meta(1), rabbit_fifo:make_checkout(Cid, {dequeue, unsettled}, #{}), State), + ok. + +untracked_enq_deq_test(_) -> + Cid = {?FUNCTION_NAME, self()}, + State0 = test_init(test), + {State1, _, _} = apply(meta(1), + rabbit_fifo:make_enqueue(undefined, undefined, first), + State0), + {_State2, {dequeue, {0, {_, first}}, _}, _} = + apply(meta(3), rabbit_fifo:make_checkout(Cid, {dequeue, settled}, #{}), State1), + ok. + +release_cursor_test(_) -> + Cid = {?FUNCTION_NAME, self()}, + {State1, _} = enq(1, 1, first, test_init(test)), + {State2, _} = enq(2, 2, second, State1), + {State3, _} = check(Cid, 3, 10, State2), + % no release cursor effect at this point + {State4, _} = settle(Cid, 4, 1, State3), + {_Final, Effects1} = settle(Cid, 5, 0, State4), + % empty queue forwards release cursor all the way + ?ASSERT_EFF({release_cursor, 5, _}, Effects1), + ok. + +checkout_enq_settle_test(_) -> + Cid = {?FUNCTION_NAME, self()}, + {State1, [{monitor, _, _} | _]} = check(Cid, 1, test_init(test)), + {State2, Effects0} = enq(2, 1, first, State1), + ?ASSERT_EFF({send_msg, _, + {delivery, ?FUNCTION_NAME, + [{0, {_, first}}]}, _}, + Effects0), + {State3, [_Inactive]} = enq(3, 2, second, State2), + {_, _Effects} = settle(Cid, 4, 0, State3), + % the release cursor is the smallest raft index that does not + % contribute to the state of the application + % ?ASSERT_EFF({release_cursor, 2, _}, Effects), + ok. + +out_of_order_enqueue_test(_) -> + Cid = {?FUNCTION_NAME, self()}, + {State1, [{monitor, _, _} | _]} = check_n(Cid, 5, 5, test_init(test)), + {State2, Effects2} = enq(2, 1, first, State1), + ?ASSERT_EFF({send_msg, _, {delivery, _, [{_, {_, first}}]}, _}, Effects2), + % assert monitor was set up + ?ASSERT_EFF({monitor, _, _}, Effects2), + % enqueue seq num 3 and 4 before 2 + {State3, Effects3} = enq(3, 3, third, State2), + ?assertNoEffect({send_msg, _, {delivery, _, _}, _}, Effects3), + {State4, Effects4} = enq(4, 4, fourth, State3), + % assert no further deliveries where made + ?assertNoEffect({send_msg, _, {delivery, _, _}, _}, Effects4), + {_State5, Effects5} = enq(5, 2, second, State4), + % assert two deliveries were now made + ?ASSERT_EFF({send_msg, _, {delivery, _, [{_, {_, second}}, + {_, {_, third}}, + {_, {_, fourth}}]}, _}, + Effects5), + ok. + +out_of_order_first_enqueue_test(_) -> + Cid = {?FUNCTION_NAME, self()}, + {State1, _} = check_n(Cid, 5, 5, test_init(test)), + {_State2, Effects2} = enq(2, 10, first, State1), + ?ASSERT_EFF({monitor, process, _}, Effects2), + ?assertNoEffect({send_msg, _, {delivery, _, [{_, {_, first}}]}, _}, + Effects2), + ok. + +duplicate_enqueue_test(_) -> + Cid = {<<"duplicate_enqueue_test">>, self()}, + {State1, [{monitor, _, _} | _]} = check_n(Cid, 5, 5, test_init(test)), + {State2, Effects2} = enq(2, 1, first, State1), + ?ASSERT_EFF({send_msg, _, {delivery, _, [{_, {_, first}}]}, _}, Effects2), + {_State3, Effects3} = enq(3, 1, first, State2), + ?assertNoEffect({send_msg, _, {delivery, _, [{_, {_, first}}]}, _}, Effects3), + ok. + +return_test(_) -> + Cid = {<<"cid">>, self()}, + Cid2 = {<<"cid2">>, self()}, + {State0, _} = enq(1, 1, msg, test_init(test)), + {State1, _} = check_auto(Cid, 2, State0), + {State2, _} = check_auto(Cid2, 3, State1), + {State3, _, _} = apply(meta(4), rabbit_fifo:make_return(Cid, [0]), State2), + ?assertMatch(#{Cid := #consumer{checked_out = C}} when map_size(C) == 0, + State3#rabbit_fifo.consumers), + ?assertMatch(#{Cid2 := #consumer{checked_out = C2}} when map_size(C2) == 1, + State3#rabbit_fifo.consumers), + ok. + +return_dequeue_delivery_limit_test(_) -> + Init = init(#{name => test, + queue_resource => rabbit_misc:r("/", queue, + atom_to_binary(test, utf8)), + release_cursor_interval => 0, + delivery_limit => 1}), + {State0, _} = enq(1, 1, msg, Init), + + Cid = {<<"cid">>, self()}, + Cid2 = {<<"cid2">>, self()}, + + {State1, {MsgId1, _}} = deq(2, Cid, unsettled, State0), + {State2, _, _} = apply(meta(4), rabbit_fifo:make_return(Cid, [MsgId1]), + State1), + + {State3, {MsgId2, _}} = deq(2, Cid2, unsettled, State2), + {State4, _, _} = apply(meta(4), rabbit_fifo:make_return(Cid2, [MsgId2]), + State3), + ?assertMatch(#{num_messages := 0}, rabbit_fifo:overview(State4)), + ok. + +return_non_existent_test(_) -> + Cid = {<<"cid">>, self()}, + {State0, [_, _Inactive]} = enq(1, 1, second, test_init(test)), + % return non-existent + {_State2, _} = apply(meta(3), rabbit_fifo:make_return(Cid, [99]), State0), + ok. + +return_checked_out_test(_) -> + Cid = {<<"cid">>, self()}, + {State0, [_, _]} = enq(1, 1, first, test_init(test)), + {State1, [_Monitor, + {send_msg, _, {delivery, _, [{MsgId, _}]}, _}, + {aux, active} | _ ]} = check_auto(Cid, 2, State0), + % returning immediately checks out the same message again + {_, ok, [{send_msg, _, {delivery, _, [{_, _}]}, _}, + {aux, active}]} = + apply(meta(3), rabbit_fifo:make_return(Cid, [MsgId]), State1), + ok. + +return_checked_out_limit_test(_) -> + Cid = {<<"cid">>, self()}, + Init = init(#{name => test, + queue_resource => rabbit_misc:r("/", queue, + atom_to_binary(test, utf8)), + release_cursor_interval => 0, + delivery_limit => 1}), + {State0, [_, _]} = enq(1, 1, first, Init), + {State1, [_Monitor, + {send_msg, _, {delivery, _, [{MsgId, _}]}, _}, + {aux, active} | _ ]} = check_auto(Cid, 2, State0), + % returning immediately checks out the same message again + {State2, ok, [{send_msg, _, {delivery, _, [{MsgId2, _}]}, _}, + {aux, active}]} = + apply(meta(3), rabbit_fifo:make_return(Cid, [MsgId]), State1), + {#rabbit_fifo{ra_indexes = RaIdxs}, ok, [_ReleaseEff]} = + apply(meta(4), rabbit_fifo:make_return(Cid, [MsgId2]), State2), + ?assertEqual(0, rabbit_fifo_index:size(RaIdxs)), + ok. + +return_auto_checked_out_test(_) -> + Cid = {<<"cid">>, self()}, + {State00, [_, _]} = enq(1, 1, first, test_init(test)), + {State0, [_]} = enq(2, 2, second, State00), + % it first active then inactive as the consumer took on but cannot take + % any more + {State1, [_Monitor, + {send_msg, _, {delivery, _, [{MsgId, _}]}, _}, + {aux, active}, + {aux, inactive} + ]} = check_auto(Cid, 2, State0), + % return should include another delivery + {_State2, _, Effects} = apply(meta(3), rabbit_fifo:make_return(Cid, [MsgId]), State1), + ?ASSERT_EFF({send_msg, _, + {delivery, _, [{_, {#{delivery_count := 1}, first}}]}, _}, + Effects), + ok. + +cancelled_checkout_out_test(_) -> + Cid = {<<"cid">>, self()}, + {State00, [_, _]} = enq(1, 1, first, test_init(test)), + {State0, [_]} = enq(2, 2, second, State00), + {State1, _} = check_auto(Cid, 2, State0), + % cancelled checkout should not return pending messages to queue + {State2, _, _} = apply(meta(3), rabbit_fifo:make_checkout(Cid, cancel, #{}), State1), + ?assertEqual(1, lqueue:len(State2#rabbit_fifo.messages)), + ?assertEqual(0, lqueue:len(State2#rabbit_fifo.returns)), + + {State3, {dequeue, empty}} = + apply(meta(3), rabbit_fifo:make_checkout(Cid, {dequeue, settled}, #{}), State2), + %% settle + {State4, ok, _} = + apply(meta(4), rabbit_fifo:make_settle(Cid, [0]), State3), + + {_State, {dequeue, {_, {_, second}}, _}, _} = + apply(meta(5), rabbit_fifo:make_checkout(Cid, {dequeue, settled}, #{}), State4), + ok. + +down_with_noproc_consumer_returns_unsettled_test(_) -> + Cid = {<<"down_consumer_returns_unsettled_test">>, self()}, + {State0, [_, _]} = enq(1, 1, second, test_init(test)), + {State1, [{monitor, process, Pid} | _]} = check(Cid, 2, State0), + {State2, _, _} = apply(meta(3), {down, Pid, noproc}, State1), + {_State, Effects} = check(Cid, 4, State2), + ?ASSERT_EFF({monitor, process, _}, Effects), + ok. + +down_with_noconnection_marks_suspect_and_node_is_monitored_test(_) -> + Pid = spawn(fun() -> ok end), + Cid = {<<"down_with_noconnect">>, Pid}, + Self = self(), + Node = node(Pid), + {State0, Effects0} = enq(1, 1, second, test_init(test)), + ?ASSERT_EFF({monitor, process, P}, P =:= Self, Effects0), + {State1, Effects1} = check_auto(Cid, 2, State0), + #consumer{credit = 0} = maps:get(Cid, State1#rabbit_fifo.consumers), + ?ASSERT_EFF({monitor, process, P}, P =:= Pid, Effects1), + % monitor both enqueuer and consumer + % because we received a noconnection we now need to monitor the node + {State2a, _, _} = apply(meta(3), {down, Pid, noconnection}, State1), + #consumer{credit = 1, + checked_out = Ch, + status = suspected_down} = maps:get(Cid, State2a#rabbit_fifo.consumers), + ?assertEqual(#{}, Ch), + %% validate consumer has credit + {State2, _, Effects2} = apply(meta(3), {down, Self, noconnection}, State2a), + ?ASSERT_EFF({monitor, node, _}, Effects2), + ?assertNoEffect({demonitor, process, _}, Effects2), + % when the node comes up we need to retry the process monitors for the + % disconnected processes + {State3, _, Effects3} = apply(meta(3), {nodeup, Node}, State2), + #consumer{status = up} = maps:get(Cid, State3#rabbit_fifo.consumers), + % try to re-monitor the suspect processes + ?ASSERT_EFF({monitor, process, P}, P =:= Pid, Effects3), + ?ASSERT_EFF({monitor, process, P}, P =:= Self, Effects3), + ok. + +down_with_noconnection_returns_unack_test(_) -> + Pid = spawn(fun() -> ok end), + Cid = {<<"down_with_noconnect">>, Pid}, + {State0, _} = enq(1, 1, second, test_init(test)), + ?assertEqual(1, lqueue:len(State0#rabbit_fifo.messages)), + ?assertEqual(0, lqueue:len(State0#rabbit_fifo.returns)), + {State1, {_, _}} = deq(2, Cid, unsettled, State0), + ?assertEqual(0, lqueue:len(State1#rabbit_fifo.messages)), + ?assertEqual(0, lqueue:len(State1#rabbit_fifo.returns)), + {State2a, _, _} = apply(meta(3), {down, Pid, noconnection}, State1), + ?assertEqual(0, lqueue:len(State2a#rabbit_fifo.messages)), + ?assertEqual(1, lqueue:len(State2a#rabbit_fifo.returns)), + ?assertMatch(#consumer{checked_out = Ch, + status = suspected_down} + when map_size(Ch) == 0, + maps:get(Cid, State2a#rabbit_fifo.consumers)), + ok. + +down_with_noproc_enqueuer_is_cleaned_up_test(_) -> + State00 = test_init(test), + Pid = spawn(fun() -> ok end), + {State0, _, Effects0} = apply(meta(1), rabbit_fifo:make_enqueue(Pid, 1, first), State00), + ?ASSERT_EFF({monitor, process, _}, Effects0), + {State1, _, _} = apply(meta(3), {down, Pid, noproc}, State0), + % ensure there are no enqueuers + ?assert(0 =:= maps:size(State1#rabbit_fifo.enqueuers)), + ok. + +discarded_message_without_dead_letter_handler_is_removed_test(_) -> + Cid = {<<"completed_consumer_yields_demonitor_effect_test">>, self()}, + {State0, [_, _]} = enq(1, 1, first, test_init(test)), + {State1, Effects1} = check_n(Cid, 2, 10, State0), + ?ASSERT_EFF({send_msg, _, + {delivery, _, [{0, {_, first}}]}, _}, + Effects1), + {_State2, _, Effects2} = apply(meta(1), + rabbit_fifo:make_discard(Cid, [0]), State1), + ?assertNoEffect({send_msg, _, + {delivery, _, [{0, {_, first}}]}, _}, + Effects2), + ok. + +discarded_message_with_dead_letter_handler_emits_log_effect_test(_) -> + Cid = {<<"completed_consumer_yields_demonitor_effect_test">>, self()}, + State00 = init(#{name => test, + queue_resource => rabbit_misc:r(<<"/">>, queue, <<"test">>), + dead_letter_handler => + {somemod, somefun, [somearg]}}), + {State0, [_, _]} = enq(1, 1, first, State00), + {State1, Effects1} = check_n(Cid, 2, 10, State0), + ?ASSERT_EFF({send_msg, _, + {delivery, _, [{0, {_, first}}]}, _}, + Effects1), + {_State2, _, Effects2} = apply(meta(1), rabbit_fifo:make_discard(Cid, [0]), State1), + % assert mod call effect with appended reason and message + ?ASSERT_EFF({log, _RaftIdxs, _}, Effects2), + ok. + +tick_test(_) -> + Cid = {<<"c">>, self()}, + Cid2 = {<<"c2">>, self()}, + {S0, _} = enq(1, 1, <<"fst">>, test_init(?FUNCTION_NAME)), + {S1, _} = enq(2, 2, <<"snd">>, S0), + {S2, {MsgId, _}} = deq(3, Cid, unsettled, S1), + {S3, {_, _}} = deq(4, Cid2, unsettled, S2), + {S4, _, _} = apply(meta(5), rabbit_fifo:make_return(Cid, [MsgId]), S3), + + [{mod_call, rabbit_quorum_queue, handle_tick, + [#resource{}, + {?FUNCTION_NAME, 1, 1, 2, 1, 3, 3}, + [_Node] + ]}] = rabbit_fifo:tick(1, S4), + ok. + + +delivery_query_returns_deliveries_test(_) -> + Tag = atom_to_binary(?FUNCTION_NAME, utf8), + Cid = {Tag, self()}, + Commands = [ + rabbit_fifo:make_checkout(Cid, {auto, 5, simple_prefetch}, #{}), + rabbit_fifo:make_enqueue(self(), 1, one), + rabbit_fifo:make_enqueue(self(), 2, two), + rabbit_fifo:make_enqueue(self(), 3, tre), + rabbit_fifo:make_enqueue(self(), 4, for) + ], + Indexes = lists:seq(1, length(Commands)), + Entries = lists:zip(Indexes, Commands), + {State, _Effects} = run_log(test_init(help), Entries), + % 3 deliveries are returned + [{0, {_, one}}] = rabbit_fifo:get_checked_out(Cid, 0, 0, State), + [_, _, _] = rabbit_fifo:get_checked_out(Cid, 1, 3, State), + ok. + +pending_enqueue_is_enqueued_on_down_test(_) -> + Cid = {<<"cid">>, self()}, + Pid = self(), + {State0, _} = enq(1, 2, first, test_init(test)), + {State1, _, _} = apply(meta(2), {down, Pid, noproc}, State0), + {_State2, {dequeue, {0, {_, first}}, 0}, _} = + apply(meta(3), rabbit_fifo:make_checkout(Cid, {dequeue, settled}, #{}), State1), + ok. + +duplicate_delivery_test(_) -> + {State0, _} = enq(1, 1, first, test_init(test)), + {#rabbit_fifo{ra_indexes = RaIdxs, + messages = Messages}, _} = enq(2, 1, first, State0), + ?assertEqual(1, rabbit_fifo_index:size(RaIdxs)), + ?assertEqual(1, lqueue:len(Messages)), + ok. + +state_enter_file_handle_leader_reservation_test(_) -> + S0 = init(#{name => the_name, + queue_resource => rabbit_misc:r(<<"/">>, queue, <<"test">>), + become_leader_handler => {m, f, [a]}}), + + Resource = {resource, <<"/">>, queue, <<"test">>}, + Effects = rabbit_fifo:state_enter(leader, S0), + ?assertEqual([ + {mod_call, m, f, [a, the_name]}, + {mod_call, rabbit_quorum_queue, file_handle_leader_reservation, [Resource]} + ], Effects), + ok. + +state_enter_file_handle_other_reservation_test(_) -> + S0 = init(#{name => the_name, + queue_resource => rabbit_misc:r(<<"/">>, queue, <<"test">>)}), + Effects = rabbit_fifo:state_enter(other, S0), + ?assertEqual([ + {mod_call, rabbit_quorum_queue, file_handle_other_reservation, []} + ], + Effects), + ok. + +state_enter_monitors_and_notifications_test(_) -> + Oth = spawn(fun () -> ok end), + {State0, _} = enq(1, 1, first, test_init(test)), + Cid = {<<"adf">>, self()}, + OthCid = {<<"oth">>, Oth}, + {State1, _} = check(Cid, 2, State0), + {State, _} = check(OthCid, 3, State1), + Self = self(), + Effects = rabbit_fifo:state_enter(leader, State), + + %% monitor all enqueuers and consumers + [{monitor, process, Self}, + {monitor, process, Oth}] = + lists:filter(fun ({monitor, process, _}) -> true; + (_) -> false + end, Effects), + [{send_msg, Self, leader_change, ra_event}, + {send_msg, Oth, leader_change, ra_event}] = + lists:filter(fun ({send_msg, _, leader_change, ra_event}) -> true; + (_) -> false + end, Effects), + ?ASSERT_EFF({monitor, process, _}, Effects), + ok. + +purge_test(_) -> + Cid = {<<"purge_test">>, self()}, + {State1, _} = enq(1, 1, first, test_init(test)), + {State2, {purge, 1}, _} = apply(meta(2), rabbit_fifo:make_purge(), State1), + {State3, _} = enq(3, 2, second, State2), + % get returns a reply value + {_State4, {dequeue, {0, {_, second}}, _}, [{monitor, _, _}]} = + apply(meta(4), rabbit_fifo:make_checkout(Cid, {dequeue, unsettled}, #{}), State3), + ok. + +purge_with_checkout_test(_) -> + Cid = {<<"purge_test">>, self()}, + {State0, _} = check_auto(Cid, 1, test_init(?FUNCTION_NAME)), + {State1, _} = enq(2, 1, <<"first">>, State0), + {State2, _} = enq(3, 2, <<"second">>, State1), + %% assert message bytes are non zero + ?assert(State2#rabbit_fifo.msg_bytes_checkout > 0), + ?assert(State2#rabbit_fifo.msg_bytes_enqueue > 0), + {State3, {purge, 1}, _} = apply(meta(2), rabbit_fifo:make_purge(), State2), + ?assert(State2#rabbit_fifo.msg_bytes_checkout > 0), + ?assertEqual(0, State3#rabbit_fifo.msg_bytes_enqueue), + ?assertEqual(1, rabbit_fifo_index:size(State3#rabbit_fifo.ra_indexes)), + #consumer{checked_out = Checked} = maps:get(Cid, State3#rabbit_fifo.consumers), + ?assertEqual(1, maps:size(Checked)), + ok. + +down_noproc_returns_checked_out_in_order_test(_) -> + S0 = test_init(?FUNCTION_NAME), + %% enqueue 100 + S1 = lists:foldl(fun (Num, FS0) -> + {FS, _} = enq(Num, Num, Num, FS0), + FS + end, S0, lists:seq(1, 100)), + ?assertEqual(100, lqueue:len(S1#rabbit_fifo.messages)), + Cid = {<<"cid">>, self()}, + {S2, _} = check(Cid, 101, 1000, S1), + #consumer{checked_out = Checked} = maps:get(Cid, S2#rabbit_fifo.consumers), + ?assertEqual(100, maps:size(Checked)), + %% simulate down + {S, _, _} = apply(meta(102), {down, self(), noproc}, S2), + Returns = lqueue:to_list(S#rabbit_fifo.returns), + ?assertEqual(100, length(Returns)), + ?assertEqual(0, maps:size(S#rabbit_fifo.consumers)), + %% validate returns are in order + ?assertEqual(lists:sort(Returns), Returns), + ok. + +down_noconnection_returns_checked_out_test(_) -> + S0 = test_init(?FUNCTION_NAME), + NumMsgs = 20, + S1 = lists:foldl(fun (Num, FS0) -> + {FS, _} = enq(Num, Num, Num, FS0), + FS + end, S0, lists:seq(1, NumMsgs)), + ?assertEqual(NumMsgs, lqueue:len(S1#rabbit_fifo.messages)), + Cid = {<<"cid">>, self()}, + {S2, _} = check(Cid, 101, 1000, S1), + #consumer{checked_out = Checked} = maps:get(Cid, S2#rabbit_fifo.consumers), + ?assertEqual(NumMsgs, maps:size(Checked)), + %% simulate down + {S, _, _} = apply(meta(102), {down, self(), noconnection}, S2), + Returns = lqueue:to_list(S#rabbit_fifo.returns), + ?assertEqual(NumMsgs, length(Returns)), + ?assertMatch(#consumer{checked_out = Ch} + when map_size(Ch) == 0, + maps:get(Cid, S#rabbit_fifo.consumers)), + %% validate returns are in order + ?assertEqual(lists:sort(Returns), Returns), + ok. + +single_active_consumer_basic_get_test(_) -> + Cid = {?FUNCTION_NAME, self()}, + State0 = init(#{name => ?FUNCTION_NAME, + queue_resource => rabbit_misc:r("/", queue, + atom_to_binary(?FUNCTION_NAME, utf8)), + release_cursor_interval => 0, + single_active_consumer_on => true}), + ?assertEqual(single_active, State0#rabbit_fifo.cfg#cfg.consumer_strategy), + ?assertEqual(0, map_size(State0#rabbit_fifo.consumers)), + {State1, _} = enq(1, 1, first, State0), + {_State, {error, {unsupported, single_active_consumer}}} = + apply(meta(2), rabbit_fifo:make_checkout(Cid, {dequeue, unsettled}, #{}), + State1), + ok. + +single_active_consumer_test(_) -> + State0 = init(#{name => ?FUNCTION_NAME, + queue_resource => rabbit_misc:r("/", queue, + atom_to_binary(?FUNCTION_NAME, utf8)), + release_cursor_interval => 0, + single_active_consumer_on => true}), + ?assertEqual(single_active, State0#rabbit_fifo.cfg#cfg.consumer_strategy), + ?assertEqual(0, map_size(State0#rabbit_fifo.consumers)), + + % adding some consumers + AddConsumer = fun(CTag, State) -> + {NewState, _, _} = apply( + meta(1), + make_checkout({CTag, self()}, + {once, 1, simple_prefetch}, + #{}), + State), + NewState + end, + State1 = lists:foldl(AddConsumer, State0, + [<<"ctag1">>, <<"ctag2">>, <<"ctag3">>, <<"ctag4">>]), + C1 = {<<"ctag1">>, self()}, + C2 = {<<"ctag2">>, self()}, + C3 = {<<"ctag3">>, self()}, + C4 = {<<"ctag4">>, self()}, + + % the first registered consumer is the active one, the others are waiting + ?assertEqual(1, map_size(State1#rabbit_fifo.consumers)), + ?assertMatch(#{C1 := _}, State1#rabbit_fifo.consumers), + ?assertEqual(3, length(State1#rabbit_fifo.waiting_consumers)), + ?assertNotEqual(false, lists:keyfind(C2, 1, State1#rabbit_fifo.waiting_consumers)), + ?assertNotEqual(false, lists:keyfind(C3, 1, State1#rabbit_fifo.waiting_consumers)), + ?assertNotEqual(false, lists:keyfind(C4, 1, State1#rabbit_fifo.waiting_consumers)), + + % cancelling a waiting consumer + {State2, _, Effects1} = apply(meta(2), + make_checkout(C3, cancel, #{}), + State1), + % the active consumer should still be in place + ?assertEqual(1, map_size(State2#rabbit_fifo.consumers)), + ?assertMatch(#{C1 := _}, State2#rabbit_fifo.consumers), + % the cancelled consumer has been removed from waiting consumers + ?assertEqual(2, length(State2#rabbit_fifo.waiting_consumers)), + ?assertNotEqual(false, lists:keyfind(C2, 1, State2#rabbit_fifo.waiting_consumers)), + ?assertNotEqual(false, lists:keyfind(C4, 1, State2#rabbit_fifo.waiting_consumers)), + % there are some effects to unregister the consumer + ?ASSERT_EFF({mod_call, rabbit_quorum_queue, + cancel_consumer_handler, [_, C]}, C == C3, Effects1), + + % cancelling the active consumer + {State3, _, Effects2} = apply(meta(3), + make_checkout(C1, cancel, #{}), + State2), + % the second registered consumer is now the active one + ?assertEqual(1, map_size(State3#rabbit_fifo.consumers)), + ?assertMatch(#{C2 := _}, State3#rabbit_fifo.consumers), + % the new active consumer is no longer in the waiting list + ?assertEqual(1, length(State3#rabbit_fifo.waiting_consumers)), + ?assertNotEqual(false, lists:keyfind(C4, 1, + State3#rabbit_fifo.waiting_consumers)), + %% should have a cancel consumer handler mod_call effect and + %% an active new consumer effect + ?ASSERT_EFF({mod_call, rabbit_quorum_queue, + cancel_consumer_handler, [_, C]}, C == C1, Effects2), + ?ASSERT_EFF({mod_call, rabbit_quorum_queue, + update_consumer_handler, _}, Effects2), + + % cancelling the active consumer + {State4, _, Effects3} = apply(meta(4), + make_checkout(C2, cancel, #{}), + State3), + % the last waiting consumer became the active one + ?assertEqual(1, map_size(State4#rabbit_fifo.consumers)), + ?assertMatch(#{C4 := _}, State4#rabbit_fifo.consumers), + % the waiting consumer list is now empty + ?assertEqual(0, length(State4#rabbit_fifo.waiting_consumers)), + % there are some effects to unregister the consumer and + % to update the new active one (metrics) + ?ASSERT_EFF({mod_call, rabbit_quorum_queue, + cancel_consumer_handler, [_, C]}, C == C2, Effects3), + ?ASSERT_EFF({mod_call, rabbit_quorum_queue, + update_consumer_handler, _}, Effects3), + + % cancelling the last consumer + {State5, _, Effects4} = apply(meta(5), + make_checkout(C4, cancel, #{}), + State4), + % no active consumer anymore + ?assertEqual(0, map_size(State5#rabbit_fifo.consumers)), + % still nothing in the waiting list + ?assertEqual(0, length(State5#rabbit_fifo.waiting_consumers)), + % there is an effect to unregister the consumer + queue inactive effect + ?ASSERT_EFF({mod_call, rabbit_quorum_queue, + cancel_consumer_handler, _}, Effects4), + + ok. + +single_active_consumer_cancel_consumer_when_channel_is_down_test(_) -> + State0 = init(#{name => ?FUNCTION_NAME, + queue_resource => rabbit_misc:r("/", queue, + atom_to_binary(?FUNCTION_NAME, utf8)), + release_cursor_interval => 0, + single_active_consumer_on => true}), + + DummyFunction = fun() -> ok end, + Pid1 = spawn(DummyFunction), + Pid2 = spawn(DummyFunction), + Pid3 = spawn(DummyFunction), + + [C1, C2, C3, C4] = Consumers = + [{<<"ctag1">>, Pid1}, {<<"ctag2">>, Pid2}, + {<<"ctag3">>, Pid2}, {<<"ctag4">>, Pid3}], + % adding some consumers + AddConsumer = fun({CTag, ChannelId}, State) -> + {NewState, _, _} = apply( + #{index => 1}, + make_checkout({CTag, ChannelId}, {once, 1, simple_prefetch}, #{}), + State), + NewState + end, + State1 = lists:foldl(AddConsumer, State0, Consumers), + + % the channel of the active consumer goes down + {State2, _, Effects} = apply(meta(2), {down, Pid1, noproc}, State1), + % fell back to another consumer + ?assertEqual(1, map_size(State2#rabbit_fifo.consumers)), + % there are still waiting consumers + ?assertEqual(2, length(State2#rabbit_fifo.waiting_consumers)), + % effects to unregister the consumer and + % to update the new active one (metrics) are there + ?ASSERT_EFF({mod_call, rabbit_quorum_queue, + cancel_consumer_handler, [_, C]}, C == C1, Effects), + ?ASSERT_EFF({mod_call, rabbit_quorum_queue, + update_consumer_handler, _}, Effects), + + % the channel of the active consumer and a waiting consumer goes down + {State3, _, Effects2} = apply(meta(3), {down, Pid2, noproc}, State2), + % fell back to another consumer + ?assertEqual(1, map_size(State3#rabbit_fifo.consumers)), + % no more waiting consumer + ?assertEqual(0, length(State3#rabbit_fifo.waiting_consumers)), + % effects to cancel both consumers of this channel + effect to update the new active one (metrics) + ?ASSERT_EFF({mod_call, rabbit_quorum_queue, + cancel_consumer_handler, [_, C]}, C == C2, Effects2), + ?ASSERT_EFF({mod_call, rabbit_quorum_queue, + cancel_consumer_handler, [_, C]}, C == C3, Effects2), + ?ASSERT_EFF({mod_call, rabbit_quorum_queue, + update_consumer_handler, _}, Effects2), + + % the last channel goes down + {State4, _, Effects3} = apply(meta(4), {down, Pid3, doesnotmatter}, State3), + % no more consumers + ?assertEqual(0, map_size(State4#rabbit_fifo.consumers)), + ?assertEqual(0, length(State4#rabbit_fifo.waiting_consumers)), + % there is an effect to unregister the consumer + queue inactive effect + ?ASSERT_EFF({mod_call, rabbit_quorum_queue, + cancel_consumer_handler, [_, C]}, C == C4, Effects3), + + ok. + +single_active_returns_messages_on_noconnection_test(_) -> + R = rabbit_misc:r("/", queue, atom_to_binary(?FUNCTION_NAME, utf8)), + State0 = init(#{name => ?FUNCTION_NAME, + queue_resource => R, + release_cursor_interval => 0, + single_active_consumer_on => true}), + Meta = #{index => 1}, + Nodes = [n1], + ConsumerIds = [{_, DownPid}] = + [begin + B = atom_to_binary(N, utf8), + {<<"ctag_", B/binary>>, + test_util:fake_pid(N)} + end || N <- Nodes], + % adding some consumers + State1 = lists:foldl( + fun(CId, Acc0) -> + {Acc, _, _} = + apply(Meta, + make_checkout(CId, + {once, 1, simple_prefetch}, #{}), + Acc0), + Acc + end, State0, ConsumerIds), + {State2, _} = enq(4, 1, msg1, State1), + % simulate node goes down + {State3, _, _} = apply(meta(5), {down, DownPid, noconnection}, State2), + %% assert the consumer is up + ?assertMatch([_], lqueue:to_list(State3#rabbit_fifo.returns)), + ?assertMatch([{_, #consumer{checked_out = Checked}}] + when map_size(Checked) == 0, + State3#rabbit_fifo.waiting_consumers), + + ok. + +single_active_consumer_replaces_consumer_when_down_noconnection_test(_) -> + R = rabbit_misc:r("/", queue, atom_to_binary(?FUNCTION_NAME, utf8)), + State0 = init(#{name => ?FUNCTION_NAME, + queue_resource => R, + release_cursor_interval => 0, + single_active_consumer_on => true}), + Meta = #{index => 1}, + Nodes = [n1, n2, node()], + ConsumerIds = [C1 = {_, DownPid}, C2, _C3] = + [begin + B = atom_to_binary(N, utf8), + {<<"ctag_", B/binary>>, + test_util:fake_pid(N)} + end || N <- Nodes], + % adding some consumers + State1a = lists:foldl( + fun(CId, Acc0) -> + {Acc, _, _} = + apply(Meta, + make_checkout(CId, + {once, 1, simple_prefetch}, #{}), + Acc0), + Acc + end, State0, ConsumerIds), + + %% assert the consumer is up + ?assertMatch(#{C1 := #consumer{status = up}}, + State1a#rabbit_fifo.consumers), + + {State1, _} = enq(10, 1, msg, State1a), + + % simulate node goes down + {State2, _, _} = apply(meta(5), {down, DownPid, noconnection}, State1), + + %% assert a new consumer is in place and it is up + ?assertMatch([{C2, #consumer{status = up, + checked_out = Ch}}] + when map_size(Ch) == 1, + maps:to_list(State2#rabbit_fifo.consumers)), + + %% the disconnected consumer has been returned to waiting + ?assert(lists:any(fun ({C,_}) -> C =:= C1 end, + State2#rabbit_fifo.waiting_consumers)), + ?assertEqual(2, length(State2#rabbit_fifo.waiting_consumers)), + + % simulate node comes back up + {State3, _, _} = apply(#{index => 2}, {nodeup, node(DownPid)}, State2), + + %% the consumer is still active and the same as before + ?assertMatch([{C2, #consumer{status = up}}], + maps:to_list(State3#rabbit_fifo.consumers)), + % the waiting consumers should be un-suspected + ?assertEqual(2, length(State3#rabbit_fifo.waiting_consumers)), + lists:foreach(fun({_, #consumer{status = Status}}) -> + ?assert(Status /= suspected_down) + end, State3#rabbit_fifo.waiting_consumers), + ok. + +single_active_consumer_all_disconnected_test(_) -> + R = rabbit_misc:r("/", queue, atom_to_binary(?FUNCTION_NAME, utf8)), + State0 = init(#{name => ?FUNCTION_NAME, + queue_resource => R, + release_cursor_interval => 0, + single_active_consumer_on => true}), + Meta = #{index => 1}, + Nodes = [n1, n2], + ConsumerIds = [C1 = {_, C1Pid}, C2 = {_, C2Pid}] = + [begin + B = atom_to_binary(N, utf8), + {<<"ctag_", B/binary>>, + test_util:fake_pid(N)} + end || N <- Nodes], + % adding some consumers + State1 = lists:foldl( + fun(CId, Acc0) -> + {Acc, _, _} = + apply(Meta, + make_checkout(CId, + {once, 1, simple_prefetch}, #{}), + Acc0), + Acc + end, State0, ConsumerIds), + + %% assert the consumer is up + ?assertMatch(#{C1 := #consumer{status = up}}, State1#rabbit_fifo.consumers), + + % simulate node goes down + {State2, _, _} = apply(meta(5), {down, C1Pid, noconnection}, State1), + %% assert the consumer fails over to the consumer on n2 + ?assertMatch(#{C2 := #consumer{status = up}}, State2#rabbit_fifo.consumers), + {State3, _, _} = apply(meta(6), {down, C2Pid, noconnection}, State2), + %% assert these no active consumer after both nodes are maked as down + ?assertMatch([], maps:to_list(State3#rabbit_fifo.consumers)), + %% n2 comes back + {State4, _, _} = apply(meta(7), {nodeup, node(C2Pid)}, State3), + %% ensure n2 is the active consumer as this node as been registered + %% as up again + ?assertMatch([{{<<"ctag_n2">>, _}, #consumer{status = up, + credit = 1}}], + maps:to_list(State4#rabbit_fifo.consumers)), + ok. + +single_active_consumer_state_enter_leader_include_waiting_consumers_test(_) -> + State0 = init(#{name => ?FUNCTION_NAME, + queue_resource => + rabbit_misc:r("/", queue, + atom_to_binary(?FUNCTION_NAME, utf8)), + release_cursor_interval => 0, + single_active_consumer_on => true}), + + DummyFunction = fun() -> ok end, + Pid1 = spawn(DummyFunction), + Pid2 = spawn(DummyFunction), + Pid3 = spawn(DummyFunction), + + Meta = #{index => 1}, + % adding some consumers + AddConsumer = fun({CTag, ChannelId}, State) -> + {NewState, _, _} = apply( + Meta, + make_checkout({CTag, ChannelId}, + {once, 1, simple_prefetch}, #{}), + State), + NewState + end, + State1 = lists:foldl(AddConsumer, State0, + [{<<"ctag1">>, Pid1}, {<<"ctag2">>, Pid2}, {<<"ctag3">>, Pid2}, {<<"ctag4">>, Pid3}]), + + Effects = rabbit_fifo:state_enter(leader, State1), + %% 2 effects for each consumer process (channel process), 1 effect for the node, + %% 1 effect for file handle reservation + ?assertEqual(2 * 3 + 1 + 1, length(Effects)). + +single_active_consumer_state_enter_eol_include_waiting_consumers_test(_) -> + Resource = rabbit_misc:r("/", queue, atom_to_binary(?FUNCTION_NAME, utf8)), + State0 = init(#{name => ?FUNCTION_NAME, + queue_resource => Resource, + release_cursor_interval => 0, + single_active_consumer_on => true}), + + DummyFunction = fun() -> ok end, + Pid1 = spawn(DummyFunction), + Pid2 = spawn(DummyFunction), + Pid3 = spawn(DummyFunction), + + Meta = #{index => 1}, + % adding some consumers + AddConsumer = fun({CTag, ChannelId}, State) -> + {NewState, _, _} = apply( + Meta, + make_checkout({CTag, ChannelId}, + {once, 1, simple_prefetch}, #{}), + State), + NewState + end, + State1 = lists:foldl(AddConsumer, State0, + [{<<"ctag1">>, Pid1}, {<<"ctag2">>, Pid2}, + {<<"ctag3">>, Pid2}, {<<"ctag4">>, Pid3}]), + + Effects = rabbit_fifo:state_enter(eol, State1), + %% 1 effect for each consumer process (channel process), + %% 1 effect for file handle reservation + ?assertEqual(4, length(Effects)). + +query_consumers_test(_) -> + State0 = init(#{name => ?FUNCTION_NAME, + queue_resource => rabbit_misc:r("/", queue, + atom_to_binary(?FUNCTION_NAME, utf8)), + release_cursor_interval => 0, + single_active_consumer_on => false}), + + % adding some consumers + AddConsumer = fun(CTag, State) -> + {NewState, _, _} = apply( + #{index => 1}, + make_checkout({CTag, self()}, + {once, 1, simple_prefetch}, #{}), + State), + NewState + end, + State1 = lists:foldl(AddConsumer, State0, [<<"ctag1">>, <<"ctag2">>, <<"ctag3">>, <<"ctag4">>]), + Consumers0 = State1#rabbit_fifo.consumers, + Consumer = maps:get({<<"ctag2">>, self()}, Consumers0), + Consumers1 = maps:put({<<"ctag2">>, self()}, + Consumer#consumer{status = suspected_down}, Consumers0), + State2 = State1#rabbit_fifo{consumers = Consumers1}, + + ?assertEqual(3, rabbit_fifo:query_consumer_count(State2)), + Consumers2 = rabbit_fifo:query_consumers(State2), + ?assertEqual(4, maps:size(Consumers2)), + maps:fold(fun(_Key, {Pid, Tag, _, _, Active, ActivityStatus, _, _}, _Acc) -> + ?assertEqual(self(), Pid), + case Tag of + <<"ctag2">> -> + ?assertNot(Active), + ?assertEqual(suspected_down, ActivityStatus); + _ -> + ?assert(Active), + ?assertEqual(up, ActivityStatus) + end + end, [], Consumers2). + +query_consumers_when_single_active_consumer_is_on_test(_) -> + State0 = init(#{name => ?FUNCTION_NAME, + queue_resource => rabbit_misc:r("/", queue, + atom_to_binary(?FUNCTION_NAME, utf8)), + release_cursor_interval => 0, + single_active_consumer_on => true}), + Meta = #{index => 1}, + % adding some consumers + AddConsumer = fun(CTag, State) -> + {NewState, _, _} = apply( + Meta, + make_checkout({CTag, self()}, + {once, 1, simple_prefetch}, #{}), + State), + NewState + end, + State1 = lists:foldl(AddConsumer, State0, [<<"ctag1">>, <<"ctag2">>, <<"ctag3">>, <<"ctag4">>]), + + ?assertEqual(4, rabbit_fifo:query_consumer_count(State1)), + Consumers = rabbit_fifo:query_consumers(State1), + ?assertEqual(4, maps:size(Consumers)), + maps:fold(fun(_Key, {Pid, Tag, _, _, Active, ActivityStatus, _, _}, _Acc) -> + ?assertEqual(self(), Pid), + case Tag of + <<"ctag1">> -> + ?assert(Active), + ?assertEqual(single_active, ActivityStatus); + _ -> + ?assertNot(Active), + ?assertEqual(waiting, ActivityStatus) + end + end, [], Consumers). + +active_flag_updated_when_consumer_suspected_unsuspected_test(_) -> + State0 = init(#{name => ?FUNCTION_NAME, + queue_resource => rabbit_misc:r("/", queue, + atom_to_binary(?FUNCTION_NAME, utf8)), + release_cursor_interval => 0, + single_active_consumer_on => false}), + + DummyFunction = fun() -> ok end, + Pid1 = spawn(DummyFunction), + Pid2 = spawn(DummyFunction), + Pid3 = spawn(DummyFunction), + + % adding some consumers + AddConsumer = fun({CTag, ChannelId}, State) -> + {NewState, _, _} = + apply( + #{index => 1}, + rabbit_fifo:make_checkout({CTag, ChannelId}, + {once, 1, simple_prefetch}, + #{}), + State), + NewState + end, + State1 = lists:foldl(AddConsumer, State0, + [{<<"ctag1">>, Pid1}, {<<"ctag2">>, Pid2}, {<<"ctag3">>, Pid2}, {<<"ctag4">>, Pid3}]), + + {State2, _, Effects2} = apply(#{index => 3, + system_time => 1500}, {down, Pid1, noconnection}, State1), + % 1 effect to update the metrics of each consumer (they belong to the same node), 1 more effect to monitor the node + ?assertEqual(4 + 1, length(Effects2)), + + {_, _, Effects3} = apply(#{index => 4}, {nodeup, node(self())}, State2), + % for each consumer: 1 effect to update the metrics, 1 effect to monitor the consumer PID + ?assertEqual(4 + 4, length(Effects3)). + +active_flag_not_updated_when_consumer_suspected_unsuspected_and_single_active_consumer_is_on_test(_) -> + State0 = init(#{name => ?FUNCTION_NAME, + queue_resource => rabbit_misc:r("/", queue, + atom_to_binary(?FUNCTION_NAME, utf8)), + release_cursor_interval => 0, + single_active_consumer_on => true}), + + DummyFunction = fun() -> ok end, + Pid1 = spawn(DummyFunction), + Pid2 = spawn(DummyFunction), + Pid3 = spawn(DummyFunction), + + % adding some consumers + AddConsumer = fun({CTag, ChannelId}, State) -> + {NewState, _, _} = apply( + #{index => 1}, + make_checkout({CTag, ChannelId}, + {once, 1, simple_prefetch}, #{}), + State), + NewState + end, + State1 = lists:foldl(AddConsumer, State0, + [{<<"ctag1">>, Pid1}, {<<"ctag2">>, Pid2}, + {<<"ctag3">>, Pid2}, {<<"ctag4">>, Pid3}]), + + {State2, _, Effects2} = apply(meta(2), {down, Pid1, noconnection}, State1), + % one monitor and one consumer status update (deactivated) + ?assertEqual(3, length(Effects2)), + + {_, _, Effects3} = apply(meta(3), {nodeup, node(self())}, State2), + % for each consumer: 1 effect to monitor the consumer PID + ?assertEqual(5, length(Effects3)). + +single_active_cancelled_with_unacked_test(_) -> + State0 = init(#{name => ?FUNCTION_NAME, + queue_resource => rabbit_misc:r("/", queue, + atom_to_binary(?FUNCTION_NAME, utf8)), + release_cursor_interval => 0, + single_active_consumer_on => true}), + + C1 = {<<"ctag1">>, self()}, + C2 = {<<"ctag2">>, self()}, + % adding some consumers + AddConsumer = fun(C, S0) -> + {S, _, _} = apply( + meta(1), + make_checkout(C, + {auto, 1, simple_prefetch}, + #{}), + S0), + S + end, + State1 = lists:foldl(AddConsumer, State0, [C1, C2]), + + %% enqueue 2 messages + {State2, _Effects2} = enq(3, 1, msg1, State1), + {State3, _Effects3} = enq(4, 2, msg2, State2), + %% one should be checked ou to C1 + %% cancel C1 + {State4, _, _} = apply(meta(5), + make_checkout(C1, cancel, #{}), + State3), + %% C2 should be the active consumer + ?assertMatch(#{C2 := #consumer{status = up, + checked_out = #{0 := _}}}, + State4#rabbit_fifo.consumers), + %% C1 should be a cancelled consumer + ?assertMatch(#{C1 := #consumer{status = cancelled, + lifetime = once, + checked_out = #{0 := _}}}, + State4#rabbit_fifo.consumers), + ?assertMatch([], State4#rabbit_fifo.waiting_consumers), + + %% Ack both messages + {State5, _Effects5} = settle(C1, 1, 0, State4), + %% C1 should now be cancelled + {State6, _Effects6} = settle(C2, 2, 0, State5), + + %% C2 should remain + ?assertMatch(#{C2 := #consumer{status = up}}, + State6#rabbit_fifo.consumers), + %% C1 should be gone + ?assertNotMatch(#{C1 := _}, + State6#rabbit_fifo.consumers), + ?assertMatch([], State6#rabbit_fifo.waiting_consumers), + ok. + +single_active_with_credited_test(_) -> + State0 = init(#{name => ?FUNCTION_NAME, + queue_resource => rabbit_misc:r("/", queue, + atom_to_binary(?FUNCTION_NAME, utf8)), + release_cursor_interval => 0, + single_active_consumer_on => true}), + + C1 = {<<"ctag1">>, self()}, + C2 = {<<"ctag2">>, self()}, + % adding some consumers + AddConsumer = fun(C, S0) -> + {S, _, _} = apply( + meta(1), + make_checkout(C, + {auto, 0, credited}, + #{}), + S0), + S + end, + State1 = lists:foldl(AddConsumer, State0, [C1, C2]), + + %% add some credit + C1Cred = rabbit_fifo:make_credit(C1, 5, 0, false), + {State2, _, _Effects2} = apply(meta(3), C1Cred, State1), + C2Cred = rabbit_fifo:make_credit(C2, 4, 0, false), + {State3, _} = apply(meta(4), C2Cred, State2), + %% both consumers should have credit + ?assertMatch(#{C1 := #consumer{credit = 5}}, + State3#rabbit_fifo.consumers), + ?assertMatch([{C2, #consumer{credit = 4}}], + State3#rabbit_fifo.waiting_consumers), + ok. + + +register_enqueuer_test(_) -> + State0 = init(#{name => ?FUNCTION_NAME, + queue_resource => rabbit_misc:r("/", queue, + atom_to_binary(?FUNCTION_NAME, utf8)), + max_length => 2, + overflow_strategy => reject_publish}), + %% simply registering should be ok when we're below limit + Pid1 = test_util:fake_pid(node()), + {State1, ok, [_]} = apply(meta(1), make_register_enqueuer(Pid1), State0), + + {State2, ok, _} = apply(meta(2), rabbit_fifo:make_enqueue(Pid1, 1, one), State1), + %% register another enqueuer shoudl be ok + Pid2 = test_util:fake_pid(node()), + {State3, ok, [_]} = apply(meta(3), make_register_enqueuer(Pid2), State2), + + {State4, ok, _} = apply(meta(4), rabbit_fifo:make_enqueue(Pid1, 2, two), State3), + {State5, ok, Efx} = apply(meta(5), rabbit_fifo:make_enqueue(Pid1, 3, three), State4), + % ct:pal("Efx ~p", [Efx]), + %% validate all registered enqueuers are notified of overflow state + ?ASSERT_EFF({send_msg, P, {queue_status, reject_publish}, [ra_event]}, P == Pid1, Efx), + ?ASSERT_EFF({send_msg, P, {queue_status, reject_publish}, [ra_event]}, P == Pid2, Efx), + + %% this time, registry should return reject_publish + {State6, reject_publish, [_]} = apply(meta(6), make_register_enqueuer( + test_util:fake_pid(node())), State5), + ?assertMatch(#{num_enqueuers := 3}, rabbit_fifo:overview(State6)), + + + %% remove two messages this should make the queue fall below the 0.8 limit + {State7, {dequeue, _, _}, _Efx7} = + apply(meta(7), + rabbit_fifo:make_checkout(<<"a">>, {dequeue, settled}, #{}), State6), + ct:pal("Efx7 ~p", [_Efx7]), + {State8, {dequeue, _, _}, Efx8} = + apply(meta(8), + rabbit_fifo:make_checkout(<<"a">>, {dequeue, settled}, #{}), State7), + ct:pal("Efx8 ~p", [Efx8]), + %% validate all registered enqueuers are notified of overflow state + ?ASSERT_EFF({send_msg, P, {queue_status, go}, [ra_event]}, P == Pid1, Efx8), + ?ASSERT_EFF({send_msg, P, {queue_status, go}, [ra_event]}, P == Pid2, Efx8), + {_State9, {dequeue, _, _}, Efx9} = + apply(meta(9), + rabbit_fifo:make_checkout(<<"a">>, {dequeue, settled}, #{}), State8), + ?ASSERT_NO_EFF({send_msg, P, go, [ra_event]}, P == Pid1, Efx9), + ?ASSERT_NO_EFF({send_msg, P, go, [ra_event]}, P == Pid2, Efx9), + ok. + +reject_publish_purge_test(_) -> + State0 = init(#{name => ?FUNCTION_NAME, + queue_resource => rabbit_misc:r("/", queue, + atom_to_binary(?FUNCTION_NAME, utf8)), + max_length => 2, + overflow_strategy => reject_publish}), + %% simply registering should be ok when we're below limit + Pid1 = test_util:fake_pid(node()), + {State1, ok, [_]} = apply(meta(1), make_register_enqueuer(Pid1), State0), + {State2, ok, _} = apply(meta(2), rabbit_fifo:make_enqueue(Pid1, 1, one), State1), + {State3, ok, _} = apply(meta(3), rabbit_fifo:make_enqueue(Pid1, 2, two), State2), + {State4, ok, Efx} = apply(meta(4), rabbit_fifo:make_enqueue(Pid1, 3, three), State3), + % ct:pal("Efx ~p", [Efx]), + ?ASSERT_EFF({send_msg, P, {queue_status, reject_publish}, [ra_event]}, P == Pid1, Efx), + {_State5, {purge, 3}, Efx1} = apply(meta(5), rabbit_fifo:make_purge(), State4), + ?ASSERT_EFF({send_msg, P, {queue_status, go}, [ra_event]}, P == Pid1, Efx1), + ok. + +reject_publish_applied_after_limit_test(_) -> + InitConf = #{name => ?FUNCTION_NAME, + queue_resource => rabbit_misc:r("/", queue, + atom_to_binary(?FUNCTION_NAME, utf8)) + }, + State0 = init(InitConf), + %% simply registering should be ok when we're below limit + Pid1 = test_util:fake_pid(node()), + {State1, ok, [_]} = apply(meta(1), make_register_enqueuer(Pid1), State0), + {State2, ok, _} = apply(meta(2), rabbit_fifo:make_enqueue(Pid1, 1, one), State1), + {State3, ok, _} = apply(meta(3), rabbit_fifo:make_enqueue(Pid1, 2, two), State2), + {State4, ok, Efx} = apply(meta(4), rabbit_fifo:make_enqueue(Pid1, 3, three), State3), + % ct:pal("Efx ~p", [Efx]), + ?ASSERT_NO_EFF({send_msg, P, {queue_status, reject_publish}, [ra_event]}, P == Pid1, Efx), + %% apply new config + Conf = #{name => ?FUNCTION_NAME, + queue_resource => rabbit_misc:r("/", queue, + atom_to_binary(?FUNCTION_NAME, utf8)), + max_length => 2, + overflow_strategy => reject_publish + }, + {State5, ok, Efx1} = apply(meta(5), rabbit_fifo:make_update_config(Conf), State4), + ?ASSERT_EFF({send_msg, P, {queue_status, reject_publish}, [ra_event]}, P == Pid1, Efx1), + Pid2 = test_util:fake_pid(node()), + {_State6, reject_publish, _} = apply(meta(1), make_register_enqueuer(Pid2), State5), + ok. + +purge_nodes_test(_) -> + Node = purged@node, + ThisNode = node(), + EnqPid = test_util:fake_pid(Node), + EnqPid2 = test_util:fake_pid(node()), + ConPid = test_util:fake_pid(Node), + Cid = {<<"tag">>, ConPid}, + % WaitingPid = test_util:fake_pid(Node), + + State0 = init(#{name => ?FUNCTION_NAME, + queue_resource => rabbit_misc:r("/", queue, + atom_to_binary(?FUNCTION_NAME, utf8)), + single_active_consumer_on => false}), + {State1, _, _} = apply(meta(1), + rabbit_fifo:make_enqueue(EnqPid, 1, msg1), + State0), + {State2, _, _} = apply(meta(2), + rabbit_fifo:make_enqueue(EnqPid2, 1, msg2), + State1), + {State3, _} = check(Cid, 3, 1000, State2), + {State4, _, _} = apply(meta(4), + {down, EnqPid, noconnection}, + State3), + ?assertMatch( + [{mod_call, rabbit_quorum_queue, handle_tick, + [#resource{}, _Metrics, + [ThisNode, Node] + ]}] , rabbit_fifo:tick(1, State4)), + %% assert there are both enqueuers and consumers + {State, _, _} = apply(meta(5), + rabbit_fifo:make_purge_nodes([Node]), + State4), + + %% assert there are no enqueuers nor consumers + ?assertMatch(#rabbit_fifo{enqueuers = Enqs} when map_size(Enqs) == 1, + State), + + ?assertMatch(#rabbit_fifo{consumers = Cons} when map_size(Cons) == 0, + State), + ?assertMatch( + [{mod_call, rabbit_quorum_queue, handle_tick, + [#resource{}, _Metrics, + [ThisNode] + ]}] , rabbit_fifo:tick(1, State)), + ok. + +meta(Idx) -> + meta(Idx, 0). + +meta(Idx, Timestamp) -> + #{index => Idx, + term => 1, + system_time => Timestamp, + from => {make_ref(), self()}}. + +enq(Idx, MsgSeq, Msg, State) -> + strip_reply( + apply(meta(Idx), rabbit_fifo:make_enqueue(self(), MsgSeq, Msg), State)). + +deq(Idx, Cid, Settlement, State0) -> + {State, {dequeue, {MsgId, Msg}, _}, _} = + apply(meta(Idx), + rabbit_fifo:make_checkout(Cid, {dequeue, Settlement}, #{}), + State0), + {State, {MsgId, Msg}}. + +check_n(Cid, Idx, N, State) -> + strip_reply( + apply(meta(Idx), + rabbit_fifo:make_checkout(Cid, {auto, N, simple_prefetch}, #{}), + State)). + +check(Cid, Idx, State) -> + strip_reply( + apply(meta(Idx), + rabbit_fifo:make_checkout(Cid, {once, 1, simple_prefetch}, #{}), + State)). + +check_auto(Cid, Idx, State) -> + strip_reply( + apply(meta(Idx), + rabbit_fifo:make_checkout(Cid, {auto, 1, simple_prefetch}, #{}), + State)). + +check(Cid, Idx, Num, State) -> + strip_reply( + apply(meta(Idx), + rabbit_fifo:make_checkout(Cid, {auto, Num, simple_prefetch}, #{}), + State)). + +settle(Cid, Idx, MsgId, State) -> + strip_reply(apply(meta(Idx), rabbit_fifo:make_settle(Cid, [MsgId]), State)). + +credit(Cid, Idx, Credit, DelCnt, Drain, State) -> + strip_reply(apply(meta(Idx), rabbit_fifo:make_credit(Cid, Credit, DelCnt, Drain), + State)). + +strip_reply({State, _, Effects}) -> + {State, Effects}. + +run_log(InitState, Entries) -> + lists:foldl(fun ({Idx, E}, {Acc0, Efx0}) -> + case apply(meta(Idx), E, Acc0) of + {Acc, _, Efx} when is_list(Efx) -> + {Acc, Efx0 ++ Efx}; + {Acc, _, Efx} -> + {Acc, Efx0 ++ [Efx]}; + {Acc, _} -> + {Acc, Efx0} + end + end, {InitState, []}, Entries). + + +%% AUX Tests + +aux_test(_) -> + _ = ra_machine_ets:start_link(), + Aux0 = init_aux(aux_test), + MacState = init(#{name => aux_test, + queue_resource => + rabbit_misc:r(<<"/">>, queue, <<"test">>)}), + ok = meck:new(ra_log, []), + Log = mock_log, + meck:expect(ra_log, last_index_term, fun (_) -> {0, 0} end), + {no_reply, Aux, mock_log} = handle_aux(leader, cast, active, Aux0, + Log, MacState), + {no_reply, _Aux, mock_log} = handle_aux(leader, cast, tick, Aux, + Log, MacState), + [X] = ets:lookup(rabbit_fifo_usage, aux_test), + meck:unload(), + ?assert(X > 0.0), + ok. + + +%% machine version conversion test + +machine_version_test(_) -> + V0 = rabbit_fifo_v0, + S0 = V0:init(#{name => ?FUNCTION_NAME, + queue_resource => rabbit_misc:r(<<"/">>, queue, <<"test">>)}), + Idx = 1, + {#rabbit_fifo{}, ok, []} = apply(meta(Idx), {machine_version, 0, 1}, S0), + + Cid = {atom_to_binary(?FUNCTION_NAME, utf8), self()}, + Entries = [ + {1, rabbit_fifo_v0:make_enqueue(self(), 1, banana)}, + {2, rabbit_fifo_v0:make_enqueue(self(), 2, apple)}, + {3, rabbit_fifo_v0:make_checkout(Cid, {auto, 1, unsettled}, #{})} + ], + {S1, _Effects} = rabbit_fifo_v0_SUITE:run_log(S0, Entries), + Self = self(), + {#rabbit_fifo{enqueuers = #{Self := #enqueuer{}}, + consumers = #{Cid := #consumer{priority = 0}}, + service_queue = S, + messages = Msgs}, ok, []} = apply(meta(Idx), + {machine_version, 0, 1}, S1), + %% validate message conversion to lqueue + ?assertEqual(1, lqueue:len(Msgs)), + ?assert(priority_queue:is_queue(S)), + ok. + +queue_ttl_test(_) -> + QName = rabbit_misc:r(<<"/">>, queue, <<"test">>), + Conf = #{name => ?FUNCTION_NAME, + queue_resource => QName, + created => 1000, + expires => 1000}, + S0 = rabbit_fifo:init(Conf), + Now = 1500, + [{mod_call, _, handle_tick, _}] = rabbit_fifo:tick(Now, S0), + %% this should delete the queue + [{mod_call, rabbit_quorum_queue, spawn_deleter, [QName]}] + = rabbit_fifo:tick(Now + 1000, S0), + %% adding a consumer should not ever trigger deletion + Cid = {<<"cid1">>, self()}, + {S1, _} = check_auto(Cid, 1, S0), + [{mod_call, _, handle_tick, _}] = rabbit_fifo:tick(Now, S1), + [{mod_call, _, handle_tick, _}] = rabbit_fifo:tick(Now + 1000, S1), + %% cancelling the consumer should then + {S2, _, _} = apply(meta(2, Now), + rabbit_fifo:make_checkout(Cid, cancel, #{}), S1), + %% last_active should have been reset when consumer was cancelled + %% last_active = 2500 + [{mod_call, _, handle_tick, _}] = rabbit_fifo:tick(Now + 1000, S2), + %% but now it should be deleted + [{mod_call, rabbit_quorum_queue, spawn_deleter, [QName]}] + = rabbit_fifo:tick(Now + 2500, S2), + + %% Same for downs + {S2D, _, _} = apply(meta(2, Now), + {down, self(), noconnection}, S1), + %% last_active should have been reset when consumer was cancelled + %% last_active = 2500 + [{mod_call, _, handle_tick, _}] = rabbit_fifo:tick(Now + 1000, S2D), + %% but now it should be deleted + [{mod_call, rabbit_quorum_queue, spawn_deleter, [QName]}] + = rabbit_fifo:tick(Now + 2500, S2D), + + %% dequeue should set last applied + {S1Deq, {dequeue, empty}} = + apply(meta(2, Now), + rabbit_fifo:make_checkout(Cid, {dequeue, unsettled}, #{}), + S0), + + [{mod_call, _, handle_tick, _}] = rabbit_fifo:tick(Now + 1000, S1Deq), + %% but now it should be deleted + [{mod_call, rabbit_quorum_queue, spawn_deleter, [QName]}] + = rabbit_fifo:tick(Now + 2500, S1Deq), + %% Enqueue message, + {E1, _, _} = apply(meta(2, Now), + rabbit_fifo:make_enqueue(self(), 1, msg1), S0), + Deq = {<<"deq1">>, self()}, + {E2, {dequeue, {MsgId, _}, _}, _} = + apply(meta(3, Now), + rabbit_fifo:make_checkout(Deq, {dequeue, unsettled}, #{}), + E1), + {E3, _, _} = apply(meta(3, Now + 1000), + rabbit_fifo:make_settle(Deq, [MsgId]), E2), + [{mod_call, _, handle_tick, _}] = rabbit_fifo:tick(Now + 1500, E3), + %% but now it should be deleted + [{mod_call, rabbit_quorum_queue, spawn_deleter, [QName]}] + = rabbit_fifo:tick(Now + 3000, E3), + + ok. + +queue_ttl_with_single_active_consumer_test(_) -> + QName = rabbit_misc:r(<<"/">>, queue, <<"test">>), + Conf = #{name => ?FUNCTION_NAME, + queue_resource => QName, + created => 1000, + expires => 1000, + single_active_consumer_on => true}, + S0 = rabbit_fifo:init(Conf), + Now = 1500, + [{mod_call, _, handle_tick, _}] = rabbit_fifo:tick(Now, S0), + %% this should delete the queue + [{mod_call, rabbit_quorum_queue, spawn_deleter, [QName]}] + = rabbit_fifo:tick(Now + 1000, S0), + %% adding a consumer should not ever trigger deletion + Cid = {<<"cid1">>, self()}, + {S1, _} = check_auto(Cid, 1, S0), + [{mod_call, _, handle_tick, _}] = rabbit_fifo:tick(Now, S1), + [{mod_call, _, handle_tick, _}] = rabbit_fifo:tick(Now + 1000, S1), + %% cancelling the consumer should then + {S2, _, _} = apply(meta(2, Now), + rabbit_fifo:make_checkout(Cid, cancel, #{}), S1), + %% last_active should have been reset when consumer was cancelled + %% last_active = 2500 + [{mod_call, _, handle_tick, _}] = rabbit_fifo:tick(Now + 1000, S2), + %% but now it should be deleted + [{mod_call, rabbit_quorum_queue, spawn_deleter, [QName]}] + = rabbit_fifo:tick(Now + 2500, S2), + + %% Same for downs + {S2D, _, _} = apply(meta(2, Now), + {down, self(), noconnection}, S1), + %% last_active should have been reset when consumer was cancelled + %% last_active = 2500 + [{mod_call, _, handle_tick, _}] = rabbit_fifo:tick(Now + 1000, S2D), + %% but now it should be deleted + [{mod_call, rabbit_quorum_queue, spawn_deleter, [QName]}] + = rabbit_fifo:tick(Now + 2500, S2D), + + ok. + +query_peek_test(_) -> + State0 = test_init(test), + ?assertEqual({error, no_message_at_pos}, rabbit_fifo:query_peek(1, State0)), + {State1, _} = enq(1, 1, first, State0), + {State2, _} = enq(2, 2, second, State1), + ?assertMatch({ok, {_, {_, first}}}, rabbit_fifo:query_peek(1, State1)), + ?assertEqual({error, no_message_at_pos}, rabbit_fifo:query_peek(2, State1)), + ?assertMatch({ok, {_, {_, first}}}, rabbit_fifo:query_peek(1, State2)), + ?assertMatch({ok, {_, {_, second}}}, rabbit_fifo:query_peek(2, State2)), + ?assertEqual({error, no_message_at_pos}, rabbit_fifo:query_peek(3, State2)), + ok. + +checkout_priority_test(_) -> + Cid = {<<"checkout_priority_test">>, self()}, + Pid = spawn(fun () -> ok end), + Cid2 = {<<"checkout_priority_test2">>, Pid}, + Args = [{<<"x-priority">>, long, 1}], + {S1, _, _} = + apply(meta(3), + rabbit_fifo:make_checkout(Cid, {once, 2, simple_prefetch}, + #{args => Args}), + test_init(test)), + {S2, _, _} = + apply(meta(3), + rabbit_fifo:make_checkout(Cid2, {once, 2, simple_prefetch}, + #{args => []}), + S1), + {S3, E3} = enq(1, 1, first, S2), + ?ASSERT_EFF({send_msg, P, {delivery, _, _}, _}, P == self(), E3), + {S4, E4} = enq(2, 2, second, S3), + ?ASSERT_EFF({send_msg, P, {delivery, _, _}, _}, P == self(), E4), + {_S5, E5} = enq(3, 3, third, S4), + ?ASSERT_EFF({send_msg, P, {delivery, _, _}, _}, P == Pid, E5), + ok. + +%% Utility + +init(Conf) -> rabbit_fifo:init(Conf). +make_register_enqueuer(Pid) -> rabbit_fifo:make_register_enqueuer(Pid). +apply(Meta, Entry, State) -> rabbit_fifo:apply(Meta, Entry, State). +init_aux(Conf) -> rabbit_fifo:init_aux(Conf). +handle_aux(S, T, C, A, L, M) -> rabbit_fifo:handle_aux(S, T, C, A, L, M). +make_checkout(C, S, M) -> rabbit_fifo:make_checkout(C, S, M). diff --git a/deps/rabbit/test/rabbit_fifo_int_SUITE.erl b/deps/rabbit/test/rabbit_fifo_int_SUITE.erl new file mode 100644 index 0000000000..37f5436dbf --- /dev/null +++ b/deps/rabbit/test/rabbit_fifo_int_SUITE.erl @@ -0,0 +1,661 @@ +-module(rabbit_fifo_int_SUITE). + +%% rabbit_fifo and rabbit_fifo_client integration suite + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("rabbit_common/include/rabbit.hrl"). + +-define(RA_EVENT_TIMEOUT, 5000). + +all() -> + [ + {group, tests} + ]. + +all_tests() -> + [ + basics, + return, + rabbit_fifo_returns_correlation, + resends_lost_command, + returns_after_down, + resends_after_lost_applied, + handles_reject_notification, + two_quick_enqueues, + detects_lost_delivery, + dequeue, + discard, + cancel_checkout, + credit, + untracked_enqueue, + flow, + test_queries, + duplicate_delivery, + usage + ]. + +groups() -> + [ + {tests, [], all_tests()} + ]. + +init_per_group(_, Config) -> + PrivDir = ?config(priv_dir, Config), + _ = application:load(ra), + ok = application:set_env(ra, data_dir, PrivDir), + application:ensure_all_started(ra), + application:ensure_all_started(lg), + Config. + +end_per_group(_, Config) -> + _ = application:stop(ra), + Config. + +init_per_testcase(TestCase, Config) -> + meck:new(rabbit_quorum_queue, [passthrough]), + meck:expect(rabbit_quorum_queue, handle_tick, fun (_, _, _) -> ok end), + meck:expect(rabbit_quorum_queue, file_handle_leader_reservation, fun (_) -> ok end), + meck:expect(rabbit_quorum_queue, file_handle_other_reservation, fun () -> ok end), + meck:expect(rabbit_quorum_queue, cancel_consumer_handler, + fun (_, _) -> ok end), + ra_server_sup_sup:remove_all(), + ServerName2 = list_to_atom(atom_to_list(TestCase) ++ "2"), + ServerName3 = list_to_atom(atom_to_list(TestCase) ++ "3"), + ClusterName = rabbit_misc:r("/", queue, atom_to_binary(TestCase, utf8)), + [ + {cluster_name, ClusterName}, + {uid, atom_to_binary(TestCase, utf8)}, + {node_id, {TestCase, node()}}, + {uid2, atom_to_binary(ServerName2, utf8)}, + {node_id2, {ServerName2, node()}}, + {uid3, atom_to_binary(ServerName3, utf8)}, + {node_id3, {ServerName3, node()}} + | Config]. + +end_per_testcase(_, Config) -> + meck:unload(), + Config. + +basics(Config) -> + ClusterName = ?config(cluster_name, Config), + ServerId = ?config(node_id, Config), + UId = ?config(uid, Config), + CustomerTag = UId, + ok = start_cluster(ClusterName, [ServerId]), + FState0 = rabbit_fifo_client:init(ClusterName, [ServerId]), + {ok, FState1} = rabbit_fifo_client:checkout(CustomerTag, 1, simple_prefetch, + #{}, FState0), + + ra_log_wal:force_roll_over(ra_log_wal), + % create segment the segment will trigger a snapshot + timer:sleep(1000), + + {ok, FState2} = rabbit_fifo_client:enqueue(one, FState1), + % process ra events + FState3 = process_ra_event(FState2, ?RA_EVENT_TIMEOUT), + + FState5 = receive + {ra_event, From, Evt} -> + case rabbit_fifo_client:handle_ra_event(From, Evt, FState3) of + {ok, FState4, + [{deliver, C, true, + [{_Qname, _QRef, MsgId, _SomBool, _Msg}]}]} -> + {S, _A} = rabbit_fifo_client:settle(C, [MsgId], FState4), + S + end + after 5000 -> + exit(await_msg_timeout) + end, + + % process settle applied notification + FState5b = process_ra_event(FState5, ?RA_EVENT_TIMEOUT), + _ = ra:stop_server(ServerId), + _ = ra:restart_server(ServerId), + + %% wait for leader change to notice server is up again + receive + {ra_event, _, {machine, leader_change}} -> ok + after 5000 -> + exit(leader_change_timeout) + end, + + {ok, FState6} = rabbit_fifo_client:enqueue(two, FState5b), + % process applied event + FState6b = process_ra_event(FState6, ?RA_EVENT_TIMEOUT), + + receive + {ra_event, Frm, E} -> + case rabbit_fifo_client:handle_ra_event(Frm, E, FState6b) of + {ok, FState7, [{deliver, Ctag, true, + [{_, _, Mid, _, two}]}]} -> + {_, _} = rabbit_fifo_client:return(Ctag, [Mid], FState7), + ok + end + after 2000 -> + exit(await_msg_timeout) + end, + ra:stop_server(ServerId), + ok. + +return(Config) -> + ClusterName = ?config(cluster_name, Config), + ServerId = ?config(node_id, Config), + ok = start_cluster(ClusterName, [ServerId]), + + F00 = rabbit_fifo_client:init(ClusterName, [ServerId]), + {ok, F0} = rabbit_fifo_client:enqueue(1, msg1, F00), + {ok, F1} = rabbit_fifo_client:enqueue(2, msg2, F0), + {_, _, F2} = process_ra_events(receive_ra_events(2, 0), F1), + {ok, _, {_, _, MsgId, _, _}, F} = rabbit_fifo_client:dequeue(<<"tag">>, unsettled, F2), + _F2 = rabbit_fifo_client:return(<<"tag">>, [MsgId], F), + + ra:stop_server(ServerId), + ok. + +rabbit_fifo_returns_correlation(Config) -> + ClusterName = ?config(cluster_name, Config), + ServerId = ?config(node_id, Config), + ok = start_cluster(ClusterName, [ServerId]), + F0 = rabbit_fifo_client:init(ClusterName, [ServerId]), + {ok, F1} = rabbit_fifo_client:enqueue(corr1, msg1, F0), + receive + {ra_event, Frm, E} -> + case rabbit_fifo_client:handle_ra_event(Frm, E, F1) of + {ok, _F2, [{settled, _, _}]} -> + ok; + Del -> + exit({unexpected, Del}) + end + after 2000 -> + exit(await_msg_timeout) + end, + ra:stop_server(ServerId), + ok. + +duplicate_delivery(Config) -> + ClusterName = ?config(cluster_name, Config), + ServerId = ?config(node_id, Config), + ok = start_cluster(ClusterName, [ServerId]), + F0 = rabbit_fifo_client:init(ClusterName, [ServerId]), + {ok, F1} = rabbit_fifo_client:checkout(<<"tag">>, 10, simple_prefetch, #{}, F0), + {ok, F2} = rabbit_fifo_client:enqueue(corr1, msg1, F1), + Fun = fun Loop(S0) -> + receive + {ra_event, Frm, E} = Evt -> + case rabbit_fifo_client:handle_ra_event(Frm, E, S0) of + {ok, S1, [{settled, _, _}]} -> + Loop(S1); + {ok, S1, _} -> + %% repeat event delivery + self() ! Evt, + %% check that then next received delivery doesn't + %% repeat or crash + receive + {ra_event, F, E1} -> + case rabbit_fifo_client:handle_ra_event( + F, E1, S1) of + {ok, S2, _} -> + S2 + end + end + end + after 2000 -> + exit(await_msg_timeout) + end + end, + Fun(F2), + ra:stop_server(ServerId), + ok. + +usage(Config) -> + ClusterName = ?config(cluster_name, Config), + ServerId = ?config(node_id, Config), + ok = start_cluster(ClusterName, [ServerId]), + F0 = rabbit_fifo_client:init(ClusterName, [ServerId]), + {ok, F1} = rabbit_fifo_client:checkout(<<"tag">>, 10, simple_prefetch, #{}, F0), + {ok, F2} = rabbit_fifo_client:enqueue(corr1, msg1, F1), + {ok, F3} = rabbit_fifo_client:enqueue(corr2, msg2, F2), + {_, _, _} = process_ra_events(receive_ra_events(2, 2), F3), + % force tick and usage stats emission + ServerId ! tick_timeout, + timer:sleep(50), + Use = rabbit_fifo:usage(element(1, ServerId)), + ra:stop_server(ServerId), + ?assert(Use > 0.0), + ok. + +resends_lost_command(Config) -> + ClusterName = ?config(cluster_name, Config), + ServerId = ?config(node_id, Config), + ok = start_cluster(ClusterName, [ServerId]), + + ok = meck:new(ra, [passthrough]), + + F0 = rabbit_fifo_client:init(ClusterName, [ServerId]), + {ok, F1} = rabbit_fifo_client:enqueue(msg1, F0), + % lose the enqueue + meck:expect(ra, pipeline_command, fun (_, _, _) -> ok end), + {ok, F2} = rabbit_fifo_client:enqueue(msg2, F1), + meck:unload(ra), + {ok, F3} = rabbit_fifo_client:enqueue(msg3, F2), + {_, _, F4} = process_ra_events(receive_ra_events(2, 0), F3), + {ok, _, {_, _, _, _, msg1}, F5} = rabbit_fifo_client:dequeue(<<"tag">>, settled, F4), + {ok, _, {_, _, _, _, msg2}, F6} = rabbit_fifo_client:dequeue(<<"tag">>, settled, F5), + {ok, _, {_, _, _, _, msg3}, _F7} = rabbit_fifo_client:dequeue(<<"tag">>, settled, F6), + ra:stop_server(ServerId), + ok. + +two_quick_enqueues(Config) -> + ClusterName = ?config(cluster_name, Config), + ServerId = ?config(node_id, Config), + ok = start_cluster(ClusterName, [ServerId]), + + F0 = rabbit_fifo_client:init(ClusterName, [ServerId]), + F1 = element(2, rabbit_fifo_client:enqueue(msg1, F0)), + {ok, F2} = rabbit_fifo_client:enqueue(msg2, F1), + _ = process_ra_events(receive_ra_events(2, 0), F2), + ra:stop_server(ServerId), + ok. + +detects_lost_delivery(Config) -> + ClusterName = ?config(cluster_name, Config), + ServerId = ?config(node_id, Config), + ok = start_cluster(ClusterName, [ServerId]), + + F000 = rabbit_fifo_client:init(ClusterName, [ServerId]), + {ok, F00} = rabbit_fifo_client:enqueue(msg1, F000), + {_, _, F0} = process_ra_events(receive_ra_events(1, 0), F00), + {ok, F1} = rabbit_fifo_client:checkout(<<"tag">>, 10, simple_prefetch, #{}, F0), + {ok, F2} = rabbit_fifo_client:enqueue(msg2, F1), + {ok, F3} = rabbit_fifo_client:enqueue(msg3, F2), + % lose first delivery + receive + {ra_event, _, {machine, {delivery, _, [{_, {_, msg1}}]}}} -> + ok + after 5000 -> + exit(await_delivery_timeout) + end, + + % assert three deliveries were received + {[_, _, _], _, _} = process_ra_events(receive_ra_events(2, 2), F3), + ra:stop_server(ServerId), + ok. + +returns_after_down(Config) -> + ClusterName = ?config(cluster_name, Config), + ServerId = ?config(node_id, Config), + ok = start_cluster(ClusterName, [ServerId]), + + F0 = rabbit_fifo_client:init(ClusterName, [ServerId]), + {ok, F1} = rabbit_fifo_client:enqueue(msg1, F0), + {_, _, F2} = process_ra_events(receive_ra_events(1, 0), F1), + % start a customer in a separate processes + % that exits after checkout + Self = self(), + _Pid = spawn(fun () -> + F = rabbit_fifo_client:init(ClusterName, [ServerId]), + {ok, _} = rabbit_fifo_client:checkout(<<"tag">>, 10, + simple_prefetch, + #{}, F), + Self ! checkout_done + end), + receive checkout_done -> ok after 1000 -> exit(checkout_done_timeout) end, + timer:sleep(1000), + % message should be available for dequeue + {ok, _, {_, _, _, _, msg1}, _} = rabbit_fifo_client:dequeue(<<"tag">>, settled, F2), + ra:stop_server(ServerId), + ok. + +resends_after_lost_applied(Config) -> + ClusterName = ?config(cluster_name, Config), + ServerId = ?config(node_id, Config), + ok = start_cluster(ClusterName, [ServerId]), + + F0 = rabbit_fifo_client:init(ClusterName, [ServerId]), + {ok, F1} = rabbit_fifo_client:enqueue(msg1, F0), + {_, _, F2} = process_ra_events(receive_ra_events(1, 0), F1), + {ok, F3} = rabbit_fifo_client:enqueue(msg2, F2), + % lose an applied event + receive + {ra_event, _, {applied, _}} -> + ok + after 500 -> + exit(await_ra_event_timeout) + end, + % send another message + {ok, F4} = rabbit_fifo_client:enqueue(msg3, F3), + {_, _, F5} = process_ra_events(receive_ra_events(1, 0), F4), + {ok, _, {_, _, _, _, msg1}, F6} = rabbit_fifo_client:dequeue(<<"tag">>, settled, F5), + {ok, _, {_, _, _, _, msg2}, F7} = rabbit_fifo_client:dequeue(<<"tag">>, settled, F6), + {ok, _, {_, _, _, _, msg3}, _F8} = rabbit_fifo_client:dequeue(<<"tag">>, settled, F7), + ra:stop_server(ServerId), + ok. + +handles_reject_notification(Config) -> + ClusterName = ?config(cluster_name, Config), + ServerId1 = ?config(node_id, Config), + ServerId2 = ?config(node_id2, Config), + UId1 = ?config(uid, Config), + CId = {UId1, self()}, + + ok = start_cluster(ClusterName, [ServerId1, ServerId2]), + _ = ra:process_command(ServerId1, + rabbit_fifo:make_checkout( + CId, + {auto, 10, simple_prefetch}, + #{})), + % reverse order - should try the first node in the list first + F0 = rabbit_fifo_client:init(ClusterName, [ServerId2, ServerId1]), + {ok, F1} = rabbit_fifo_client:enqueue(one, F0), + + timer:sleep(500), + + % the applied notification + _F2 = process_ra_events(receive_ra_events(1, 0), F1), + ra:stop_server(ServerId1), + ra:stop_server(ServerId2), + ok. + +discard(Config) -> + PrivDir = ?config(priv_dir, Config), + ServerId = ?config(node_id, Config), + UId = ?config(uid, Config), + ClusterName = ?config(cluster_name, Config), + Conf = #{cluster_name => ClusterName#resource.name, + id => ServerId, + uid => UId, + log_init_args => #{data_dir => PrivDir, uid => UId}, + initial_member => [], + machine => {module, rabbit_fifo, + #{queue_resource => discard, + dead_letter_handler => + {?MODULE, dead_letter_handler, [self()]}}}}, + _ = ra:start_server(Conf), + ok = ra:trigger_election(ServerId), + _ = ra:members(ServerId), + + F0 = rabbit_fifo_client:init(ClusterName, [ServerId]), + {ok, F1} = rabbit_fifo_client:checkout(<<"tag">>, 10, + simple_prefetch, #{}, F0), + {ok, F2} = rabbit_fifo_client:enqueue(msg1, F1), + F3 = discard_next_delivery(F2, 5000), + {empty, _F4} = rabbit_fifo_client:dequeue(<<"tag1">>, settled, F3), + receive + {dead_letter, Letters} -> + [{_, msg1}] = Letters, + ok + after 500 -> + flush(), + exit(dead_letter_timeout) + end, + ra:stop_server(ServerId), + ok. + +cancel_checkout(Config) -> + ClusterName = ?config(cluster_name, Config), + ServerId = ?config(node_id, Config), + ok = start_cluster(ClusterName, [ServerId]), + F0 = rabbit_fifo_client:init(ClusterName, [ServerId], 4), + {ok, F1} = rabbit_fifo_client:enqueue(m1, F0), + {ok, F2} = rabbit_fifo_client:checkout(<<"tag">>, 10, simple_prefetch, #{}, F1), + {_, _, F3} = process_ra_events(receive_ra_events(1, 1), F2, [], [], fun (_, S) -> S end), + {ok, F4} = rabbit_fifo_client:cancel_checkout(<<"tag">>, F3), + {F5, _} = rabbit_fifo_client:return(<<"tag">>, [0], F4), + {ok, _, {_, _, _, _, m1}, F5} = rabbit_fifo_client:dequeue(<<"d1">>, settled, F5), + ok. + +credit(Config) -> + ClusterName = ?config(cluster_name, Config), + ServerId = ?config(node_id, Config), + ok = start_cluster(ClusterName, [ServerId]), + F0 = rabbit_fifo_client:init(ClusterName, [ServerId], 4), + {ok, F1} = rabbit_fifo_client:enqueue(m1, F0), + {ok, F2} = rabbit_fifo_client:enqueue(m2, F1), + {_, _, F3} = process_ra_events(receive_ra_events(2, 0), F2), + %% checkout with 0 prefetch + {ok, F4} = rabbit_fifo_client:checkout(<<"tag">>, 0, credited, #{}, F3), + %% assert no deliveries + {_, _, F5} = process_ra_events(receive_ra_events(), F4, [], [], + fun + (D, _) -> error({unexpected_delivery, D}) + end), + %% provide some credit + {F6, []} = rabbit_fifo_client:credit(<<"tag">>, 1, false, F5), + {[{_, _, _, _, m1}], [{send_credit_reply, _}], F7} = + process_ra_events(receive_ra_events(1, 1), F6), + + %% credit and drain + {F8, []} = rabbit_fifo_client:credit(<<"tag">>, 4, true, F7), + {[{_, _, _, _, m2}], [{send_credit_reply, _}, {send_drained, _}], F9} = + process_ra_events(receive_ra_events(1, 1), F8), + flush(), + + %% enqueue another message - at this point the consumer credit should be + %% all used up due to the drain + {ok, F10} = rabbit_fifo_client:enqueue(m3, F9), + %% assert no deliveries + {_, _, F11} = process_ra_events(receive_ra_events(), F10, [], [], + fun + (D, _) -> error({unexpected_delivery, D}) + end), + %% credit again and receive the last message + {F12, []} = rabbit_fifo_client:credit(<<"tag">>, 10, false, F11), + {[{_, _, _, _, m3}], _, _} = process_ra_events(receive_ra_events(1, 1), F12), + ok. + +untracked_enqueue(Config) -> + ClusterName = ?config(cluster_name, Config), + ServerId = ?config(node_id, Config), + ok = start_cluster(ClusterName, [ServerId]), + + ok = rabbit_fifo_client:untracked_enqueue([ServerId], msg1), + timer:sleep(100), + F0 = rabbit_fifo_client:init(ClusterName, [ServerId]), + {ok, _, {_, _, _, _, msg1}, _F5} = rabbit_fifo_client:dequeue(<<"tag">>, settled, F0), + ra:stop_server(ServerId), + ok. + + +flow(Config) -> + ClusterName = ?config(cluster_name, Config), + ServerId = ?config(node_id, Config), + ok = start_cluster(ClusterName, [ServerId]), + F0 = rabbit_fifo_client:init(ClusterName, [ServerId], 3), + {ok, F1} = rabbit_fifo_client:enqueue(m1, F0), + {ok, F2} = rabbit_fifo_client:enqueue(m2, F1), + {ok, F3} = rabbit_fifo_client:enqueue(m3, F2), + {slow, F4} = rabbit_fifo_client:enqueue(m4, F3), + {_, _, F5} = process_ra_events(receive_ra_events(4, 0), F4), + {ok, _} = rabbit_fifo_client:enqueue(m5, F5), + ra:stop_server(ServerId), + ok. + +test_queries(Config) -> + % ok = logger:set_primary_config(level, all), + ClusterName = ?config(cluster_name, Config), + ServerId = ?config(node_id, Config), + ok = start_cluster(ClusterName, [ServerId]), + Self = self(), + P = spawn(fun () -> + F0 = rabbit_fifo_client:init(ClusterName, [ServerId], 4), + {ok, F1} = rabbit_fifo_client:enqueue(m1, F0), + {ok, F2} = rabbit_fifo_client:enqueue(m2, F1), + process_ra_events(receive_ra_events(2, 0), F2), + Self ! ready, + receive stop -> ok end + end), + receive + ready -> ok + after 5000 -> + exit(ready_timeout) + end, + F0 = rabbit_fifo_client:init(ClusterName, [ServerId], 4), + {ok, _} = rabbit_fifo_client:checkout(<<"tag">>, 1, simple_prefetch, #{}, F0), + {ok, {_, Ready}, _} = ra:local_query(ServerId, + fun rabbit_fifo:query_messages_ready/1), + ?assertEqual(1, Ready), + {ok, {_, Checked}, _} = ra:local_query(ServerId, + fun rabbit_fifo:query_messages_checked_out/1), + ?assertEqual(1, Checked), + {ok, {_, Processes}, _} = ra:local_query(ServerId, + fun rabbit_fifo:query_processes/1), + ?assertEqual(2, length(Processes)), + P ! stop, + ra:stop_server(ServerId), + ok. + +dead_letter_handler(Pid, Msgs) -> + Pid ! {dead_letter, Msgs}. + +dequeue(Config) -> + ClusterName = ?config(cluster_name, Config), + ServerId = ?config(node_id, Config), + UId = ?config(uid, Config), + Tag = UId, + ok = start_cluster(ClusterName, [ServerId]), + F1 = rabbit_fifo_client:init(ClusterName, [ServerId]), + {empty, F1b} = rabbit_fifo_client:dequeue(Tag, settled, F1), + {ok, F2_} = rabbit_fifo_client:enqueue(msg1, F1b), + {_, _, F2} = process_ra_events(receive_ra_events(1, 0), F2_), + + % {ok, {{0, {_, msg1}}, _}, F3} = rabbit_fifo_client:dequeue(Tag, settled, F2), + {ok, _, {_, _, 0, _, msg1}, F3} = rabbit_fifo_client:dequeue(Tag, settled, F2), + {ok, F4_} = rabbit_fifo_client:enqueue(msg2, F3), + {_, _, F4} = process_ra_events(receive_ra_events(1, 0), F4_), + {ok, _, {_, _, MsgId, _, msg2}, F5} = rabbit_fifo_client:dequeue(Tag, unsettled, F4), + {_F6, _A} = rabbit_fifo_client:settle(Tag, [MsgId], F5), + ra:stop_server(ServerId), + ok. + +conf(ClusterName, UId, ServerId, _, Peers) -> + #{cluster_name => ClusterName, + id => ServerId, + uid => UId, + log_init_args => #{uid => UId}, + initial_members => Peers, + machine => {module, rabbit_fifo, #{}}}. + +process_ra_event(State, Wait) -> + receive + {ra_event, From, Evt} -> + {ok, S, _Actions} = + rabbit_fifo_client:handle_ra_event(From, Evt, State), + S + after Wait -> + exit(ra_event_timeout) + end. + +receive_ra_events(Applied, Deliveries) -> + receive_ra_events(Applied, Deliveries, []). + +receive_ra_events(Applied, Deliveries, Acc) when Applied =< 0, Deliveries =< 0-> + %% what if we get more events? Testcases should check what they're! + lists:reverse(Acc); +receive_ra_events(Applied, Deliveries, Acc) -> + receive + {ra_event, _, {applied, Seqs}} = Evt -> + receive_ra_events(Applied - length(Seqs), Deliveries, [Evt | Acc]); + {ra_event, _, {machine, {delivery, _, MsgIds}}} = Evt -> + receive_ra_events(Applied, Deliveries - length(MsgIds), [Evt | Acc]); + {ra_event, _, _} = Evt -> + receive_ra_events(Applied, Deliveries, [Evt | Acc]) + after 5000 -> + exit({missing_events, Applied, Deliveries, Acc}) + end. + +%% Flusing the mailbox to later check that deliveries hasn't been received +receive_ra_events() -> + receive_ra_events([]). + +receive_ra_events(Acc) -> + receive + {ra_event, _, _} = Evt -> + receive_ra_events([Evt | Acc]) + after 500 -> + Acc + end. + +process_ra_events(Events, State) -> + DeliveryFun = fun ({deliver, _, Tag, Msgs}, S) -> + MsgIds = [element(1, M) || M <- Msgs], + {S0, _} = rabbit_fifo_client:settle(Tag, MsgIds, S), + S0 + end, + process_ra_events(Events, State, [], [], DeliveryFun). + +process_ra_events([], State0, Acc, Actions0, _DeliveryFun) -> + {Acc, Actions0, State0}; +process_ra_events([{ra_event, From, Evt} | Events], State0, Acc, Actions0, DeliveryFun) -> + case rabbit_fifo_client:handle_ra_event(From, Evt, State0) of + {ok, State1, Actions1} -> + {Msgs, Actions, State} = + lists:foldl( + fun ({deliver, _, _, Msgs} = Del, {M, A, S}) -> + {M ++ Msgs, A, DeliveryFun(Del, S)}; + (Ac, {M, A, S}) -> + {M, A ++ [Ac], S} + end, {Acc, [], State1}, Actions1), + process_ra_events(Events, State, Msgs, Actions0 ++ Actions, DeliveryFun); + eol -> + eol + end. + +discard_next_delivery(State0, Wait) -> + receive + {ra_event, _, {machine, {delivery, _, _}}} = Evt -> + element(3, process_ra_events([Evt], State0, [], [], + fun ({deliver, Tag, _, Msgs}, S) -> + MsgIds = [element(3, M) || M <- Msgs], + {S0, _} = rabbit_fifo_client:discard(Tag, MsgIds, S), + S0 + end)) + after Wait -> + State0 + end. + +return_next_delivery(State0, Wait) -> + receive + {ra_event, _, {machine, {delivery, _, _}}} = Evt -> + element(3, process_ra_events([Evt], State0, [], [], + fun ({deliver, Tag, _, Msgs}, S) -> + MsgIds = [element(3, M) || M <- Msgs], + {S0, _} = rabbit_fifo_client:return(Tag, MsgIds, S), + S0 + end)) + after Wait -> + State0 + end. + +validate_process_down(Name, 0) -> + exit({process_not_down, Name}); +validate_process_down(Name, Num) -> + case whereis(Name) of + undefined -> + ok; + _ -> + timer:sleep(100), + validate_process_down(Name, Num-1) + end. + +start_cluster(ClusterName, ServerIds, RaFifoConfig) -> + {ok, Started, _} = ra:start_cluster(ClusterName#resource.name, + {module, rabbit_fifo, RaFifoConfig}, + ServerIds), + ?assertEqual(length(Started), length(ServerIds)), + ok. + +start_cluster(ClusterName, ServerIds) -> + start_cluster(ClusterName, ServerIds, #{name => some_name, + queue_resource => ClusterName}). + +flush() -> + receive + Msg -> + ct:pal("flushed: ~w~n", [Msg]), + flush() + after 10 -> + ok + end. diff --git a/deps/rabbit/test/rabbit_fifo_prop_SUITE.erl b/deps/rabbit/test/rabbit_fifo_prop_SUITE.erl new file mode 100644 index 0000000000..859db2178f --- /dev/null +++ b/deps/rabbit/test/rabbit_fifo_prop_SUITE.erl @@ -0,0 +1,1211 @@ +-module(rabbit_fifo_prop_SUITE). + +-compile(export_all). + +-export([ + ]). + +-include_lib("proper/include/proper.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include("src/rabbit_fifo.hrl"). + +%%%=================================================================== +%%% Common Test callbacks +%%%=================================================================== + +all() -> + [ + {group, tests} + ]. + + +all_tests() -> + [ + test_run_log, + snapshots, + scenario1, + scenario2, + scenario3, + scenario4, + scenario5, + scenario6, + scenario7, + scenario8, + scenario9, + scenario10, + scenario11, + scenario12, + scenario13, + scenario14, + scenario15, + scenario16, + scenario17, + scenario18, + scenario19, + scenario20, + scenario21, + scenario22, + single_active, + single_active_01, + single_active_02, + single_active_03, + single_active_ordering, + single_active_ordering_01, + single_active_ordering_03, + in_memory_limit, + max_length + % single_active_ordering_02 + ]. + +groups() -> + [ + {tests, [], all_tests()} + ]. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_Group, Config) -> + Config. + +end_per_group(_Group, _Config) -> + ok. + +init_per_testcase(_TestCase, Config) -> + Config. + +end_per_testcase(_TestCase, _Config) -> + ok. + +%%%=================================================================== +%%% Test cases +%%%=================================================================== + +% -type log_op() :: +% {enqueue, pid(), maybe(msg_seqno()), Msg :: raw_msg()}. + +scenario1(_Config) -> + C1 = {<<>>, c:pid(0,6723,1)}, + C2 = {<<0>>,c:pid(0,6723,1)}, + E = c:pid(0,6720,1), + + Commands = [ + make_checkout(C1, {auto,2,simple_prefetch}), + make_enqueue(E,1,msg1), + make_enqueue(E,2,msg2), + make_checkout(C1, cancel), %% both on returns queue + make_checkout(C2, {auto,1,simple_prefetch}), + make_return(C2, [0]), %% E1 in returns, E2 with C2 + make_return(C2, [1]), %% E2 in returns E1 with C2 + make_settle(C2, [2]) %% E2 with C2 + ], + run_snapshot_test(#{name => ?FUNCTION_NAME}, Commands), + ok. + +scenario2(_Config) -> + C1 = {<<>>, c:pid(0,346,1)}, + C2 = {<<>>,c:pid(0,379,1)}, + E = c:pid(0,327,1), + Commands = [make_checkout(C1, {auto,1,simple_prefetch}), + make_enqueue(E,1,msg1), + make_checkout(C1, cancel), + make_enqueue(E,2,msg2), + make_checkout(C2, {auto,1,simple_prefetch}), + make_settle(C1, [0]), + make_settle(C2, [0]) + ], + run_snapshot_test(#{name => ?FUNCTION_NAME}, Commands), + ok. + +scenario3(_Config) -> + C1 = {<<>>, c:pid(0,179,1)}, + E = c:pid(0,176,1), + Commands = [make_checkout(C1, {auto,2,simple_prefetch}), + make_enqueue(E,1,msg1), + make_return(C1, [0]), + make_enqueue(E,2,msg2), + make_enqueue(E,3,msg3), + make_settle(C1, [1]), + make_settle(C1, [2]) + ], + run_snapshot_test(#{name => ?FUNCTION_NAME}, Commands), + ok. + +scenario4(_Config) -> + C1 = {<<>>, c:pid(0,179,1)}, + E = c:pid(0,176,1), + Commands = [make_checkout(C1, {auto,1,simple_prefetch}), + make_enqueue(E,1,msg), + make_settle(C1, [0]) + ], + run_snapshot_test(#{name => ?FUNCTION_NAME}, Commands), + ok. + +scenario5(_Config) -> + C1 = {<<>>, c:pid(0,505,0)}, + E = c:pid(0,465,9), + Commands = [make_enqueue(E,1,<<0>>), + make_checkout(C1, {auto,1,simple_prefetch}), + make_enqueue(E,2,<<>>), + make_settle(C1,[0])], + run_snapshot_test(#{name => ?FUNCTION_NAME}, Commands), + ok. + +scenario6(_Config) -> + E = c:pid(0,465,9), + Commands = [make_enqueue(E,1,<<>>), %% 1 msg on queue - snap: prefix 1 + make_enqueue(E,2,<<>>) %% 1. msg on queue - snap: prefix 1 + ], + run_snapshot_test(#{name => ?FUNCTION_NAME, + max_length => 1}, Commands), + ok. + +scenario7(_Config) -> + C1 = {<<>>, c:pid(0,208,0)}, + E = c:pid(0,188,0), + Commands = [ + make_enqueue(E,1,<<>>), + make_checkout(C1, {auto,1,simple_prefetch}), + make_enqueue(E,2,<<>>), + make_enqueue(E,3,<<>>), + make_settle(C1,[0])], + run_snapshot_test(#{name => ?FUNCTION_NAME, + max_length => 1}, Commands), + ok. + +scenario8(_Config) -> + C1 = {<<>>, c:pid(0,208,0)}, + E = c:pid(0,188,0), + Commands = [ + make_enqueue(E,1,<<>>), + make_enqueue(E,2,<<>>), + make_checkout(C1, {auto,1,simple_prefetch}), + % make_checkout(C1, cancel), + {down, E, noconnection}, + make_settle(C1, [0])], + run_snapshot_test(#{name => ?FUNCTION_NAME, + max_length => 1}, Commands), + ok. + +scenario9(_Config) -> + E = c:pid(0,188,0), + Commands = [ + make_enqueue(E,1,<<>>), + make_enqueue(E,2,<<>>), + make_enqueue(E,3,<<>>)], + run_snapshot_test(#{name => ?FUNCTION_NAME, + max_length => 1}, Commands), + ok. + +scenario10(_Config) -> + C1 = {<<>>, c:pid(0,208,0)}, + E = c:pid(0,188,0), + Commands = [ + make_checkout(C1, {auto,1,simple_prefetch}), + make_enqueue(E,1,<<>>), + make_settle(C1, [0]) + ], + run_snapshot_test(#{name => ?FUNCTION_NAME, + max_length => 1}, Commands), + ok. + +scenario11(_Config) -> + C1 = {<<>>, c:pid(0,215,0)}, + E = c:pid(0,217,0), + Commands = [ + make_enqueue(E,1,<<>>), + make_checkout(C1, {auto,1,simple_prefetch}), + make_checkout(C1, cancel), + make_enqueue(E,2,<<>>), + make_checkout(C1, {auto,1,simple_prefetch}), + make_settle(C1, [0]), + make_checkout(C1, cancel) + ], + run_snapshot_test(#{name => ?FUNCTION_NAME, + max_length => 2}, Commands), + ok. + +scenario12(_Config) -> + E = c:pid(0,217,0), + Commands = [make_enqueue(E,1,<<0>>), + make_enqueue(E,2,<<0>>), + make_enqueue(E,3,<<0>>)], + run_snapshot_test(#{name => ?FUNCTION_NAME, + max_bytes => 2}, Commands), + ok. + +scenario13(_Config) -> + E = c:pid(0,217,0), + Commands = [make_enqueue(E,1,<<0>>), + make_enqueue(E,2,<<>>), + make_enqueue(E,3,<<>>), + make_enqueue(E,4,<<>>) + ], + run_snapshot_test(#{name => ?FUNCTION_NAME, + max_length => 2}, Commands), + ok. + +scenario14(_Config) -> + E = c:pid(0,217,0), + Commands = [make_enqueue(E,1,<<0,0>>)], + run_snapshot_test(#{name => ?FUNCTION_NAME, + max_bytes => 1}, Commands), + ok. + +scenario15(_Config) -> + C1 = {<<>>, c:pid(0,179,1)}, + E = c:pid(0,176,1), + Commands = [make_checkout(C1, {auto,2,simple_prefetch}), + make_enqueue(E, 1, msg1), + make_enqueue(E, 2, msg2), + make_return(C1, [0]), + make_return(C1, [2]), + make_settle(C1, [1]) + ], + run_snapshot_test(#{name => ?FUNCTION_NAME, + delivery_limit => 1}, Commands), + ok. + +scenario16(_Config) -> + C1Pid = c:pid(0,883,1), + C1 = {<<>>, C1Pid}, + C2 = {<<>>, c:pid(0,882,1)}, + E = c:pid(0,176,1), + Commands = [ + make_checkout(C1, {auto,1,simple_prefetch}), + make_enqueue(E, 1, msg1), + make_checkout(C2, {auto,1,simple_prefetch}), + {down, C1Pid, noproc}, %% msg1 allocated to C2 + make_return(C2, [0]), %% msg1 returned + make_enqueue(E, 2, <<>>), + make_settle(C2, [0]) + ], + run_snapshot_test(#{name => ?FUNCTION_NAME, + delivery_limit => 1}, Commands), + ok. + +scenario17(_Config) -> + C1Pid = test_util:fake_pid(rabbit@fake_node1), + C1 = {<<0>>, C1Pid}, + % C2Pid = test_util:fake_pid(fake_node1), + C2 = {<<>>, C1Pid}, + E = test_util:fake_pid(rabbit@fake_node2), + Commands = [ + make_checkout(C1, {auto,1,simple_prefetch}), + make_enqueue(E,1,<<"one">>), + make_checkout(C2, {auto,1,simple_prefetch}), + {down, C1Pid, noconnection}, + make_checkout(C2, cancel), + make_enqueue(E,2,<<"two">>), + {nodeup,rabbit@fake_node1}, + %% this has no effect as was returned + make_settle(C1, [0]), + %% this should settle "one" + make_settle(C1, [1]) + ], + run_snapshot_test(#{name => ?FUNCTION_NAME, + single_active_consumer_on => true + }, Commands), + ok. + +scenario18(_Config) -> + E = c:pid(0,176,1), + Commands = [make_enqueue(E,1,<<"1">>), + make_enqueue(E,2,<<"2">>), + make_enqueue(E,3,<<"3">>), + make_enqueue(E,4,<<"4">>), + make_enqueue(E,5,<<"5">>) + ], + run_snapshot_test(#{name => ?FUNCTION_NAME, + %% max_length => 3, + max_in_memory_length => 1}, Commands), + ok. + +scenario19(_Config) -> + C1Pid = c:pid(0,883,1), + C1 = {<<>>, C1Pid}, + E = c:pid(0,176,1), + Commands = [make_enqueue(E,1,<<"1">>), + make_enqueue(E,2,<<"2">>), + make_checkout(C1, {auto,2,simple_prefetch}), + make_enqueue(E,3,<<"3">>), + make_settle(C1, [0, 1]) + ], + run_snapshot_test(#{name => ?FUNCTION_NAME, + max_in_memory_bytes => 370, + max_in_memory_length => 1}, Commands), + ok. + +scenario20(_Config) -> + C1Pid = c:pid(0,883,1), + C1 = {<<>>, C1Pid}, + E = c:pid(0,176,1), + Commands = [make_enqueue(E,1,<<>>), + make_enqueue(E,2,<<>>), + make_checkout(C1, {auto,2,simple_prefetch}), + {down, C1Pid, noconnection}, + make_enqueue(E,3,<<0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0>>), + make_enqueue(E,4,<<0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0>>), + make_enqueue(E,5,<<0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0>>), + make_enqueue(E,6,<<0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0>>), + make_enqueue(E,7,<<0,0,0,0,0,0,0,0,0,0,0,0,0,0>>) + ], + run_snapshot_test(#{name => ?FUNCTION_NAME, + max_bytes => 97, + max_in_memory_length => 1}, Commands), + ok. + +scenario21(_Config) -> + C1Pid = c:pid(0,883,1), + C1 = {<<>>, C1Pid}, + E = c:pid(0,176,1), + Commands = [ + make_checkout(C1, {auto,2,simple_prefetch}), + make_enqueue(E,1,<<"1">>), + make_enqueue(E,2,<<"2">>), + make_enqueue(E,3,<<"3">>), + rabbit_fifo:make_discard(C1, [0]), + rabbit_fifo:make_settle(C1, [1]) + ], + run_snapshot_test(#{name => ?FUNCTION_NAME, + release_cursor_interval => 1, + dead_letter_handler => {?MODULE, banana, []}}, + Commands), + ok. + +scenario22(_Config) -> + % C1Pid = c:pid(0,883,1), + % C1 = {<<>>, C1Pid}, + E = c:pid(0,176,1), + Commands = [ + make_enqueue(E,1,<<"1">>), + make_enqueue(E,2,<<"2">>), + make_enqueue(E,3,<<"3">>), + make_enqueue(E,4,<<"4">>), + make_enqueue(E,5,<<"5">>) + ], + run_snapshot_test(#{name => ?FUNCTION_NAME, + release_cursor_interval => 1, + max_length => 3, + dead_letter_handler => {?MODULE, banana, []}}, + Commands), + ok. + +single_active_01(_Config) -> + C1Pid = test_util:fake_pid(rabbit@fake_node1), + C1 = {<<0>>, C1Pid}, + C2Pid = test_util:fake_pid(rabbit@fake_node2), + C2 = {<<>>, C2Pid}, + E = test_util:fake_pid(rabbit@fake_node2), + Commands = [ + make_checkout(C1, {auto,1,simple_prefetch}), + make_enqueue(E,1,<<"one">>), + make_checkout(C2, {auto,1,simple_prefetch}), + make_checkout(C1, cancel), + {nodeup,rabbit@fake_node1} + ], + ?assert( + single_active_prop(#{name => ?FUNCTION_NAME, + single_active_consumer_on => true + }, Commands, false)), + ok. + +single_active_02(_Config) -> + C1Pid = test_util:fake_pid(node()), + C1 = {<<0>>, C1Pid}, + C2Pid = test_util:fake_pid(node()), + C2 = {<<>>, C2Pid}, + E = test_util:fake_pid(node()), + Commands = [ + make_checkout(C1, {auto,1,simple_prefetch}), + make_enqueue(E,1,<<"one">>), + {down,E,noconnection}, + make_checkout(C2, {auto,1,simple_prefetch}), + make_checkout(C2, cancel), + {down,E,noconnection} + ], + Conf = config(?FUNCTION_NAME, undefined, undefined, true, 1, undefined, undefined), + ?assert(single_active_prop(Conf, Commands, false)), + ok. + +single_active_03(_Config) -> + C1Pid = test_util:fake_pid(node()), + C1 = {<<0>>, C1Pid}, + % C2Pid = test_util:fake_pid(rabbit@fake_node2), + % C2 = {<<>>, C2Pid}, + Pid = test_util:fake_pid(node()), + E = test_util:fake_pid(rabbit@fake_node2), + Commands = [ + make_checkout(C1, {auto,2,simple_prefetch}), + make_enqueue(E, 1, 0), + make_enqueue(E, 2, 1), + {down, Pid, noconnection}, + {nodeup, node()} + ], + Conf = config(?FUNCTION_NAME, 0, 0, true, 0, undefined, undefined), + ?assert(single_active_prop(Conf, Commands, true)), + ok. + +test_run_log(_Config) -> + Fun = {-1, fun ({Prev, _}) -> {Prev + 1, Prev + 1} end}, + run_proper( + fun () -> + ?FORALL({Length, Bytes, SingleActiveConsumer, DeliveryLimit, InMemoryLength, + InMemoryBytes}, + frequency([{10, {0, 0, false, 0, 0, 0}}, + {5, {oneof([range(1, 10), undefined]), + oneof([range(1, 1000), undefined]), + boolean(), + oneof([range(1, 3), undefined]), + oneof([range(1, 10), undefined]), + oneof([range(1, 1000), undefined]) + }}]), + ?FORALL(O, ?LET(Ops, log_gen(100), expand(Ops, Fun)), + collect({log_size, length(O)}, + dump_generated( + config(?FUNCTION_NAME, + Length, + Bytes, + SingleActiveConsumer, + DeliveryLimit, + InMemoryLength, + InMemoryBytes), O)))) + end, [], 10). + +snapshots(_Config) -> + run_proper( + fun () -> + ?FORALL({Length, Bytes, SingleActiveConsumer, + DeliveryLimit, InMemoryLength, InMemoryBytes, + Overflow}, + frequency([{10, {0, 0, false, 0, 0, 0, drop_head}}, + {5, {oneof([range(1, 10), undefined]), + oneof([range(1, 1000), undefined]), + boolean(), + oneof([range(1, 3), undefined]), + oneof([range(1, 10), undefined]), + oneof([range(1, 1000), undefined]), + oneof([drop_head, reject_publish]) + }}]), + begin + Config = config(?FUNCTION_NAME, + Length, + Bytes, + SingleActiveConsumer, + DeliveryLimit, + InMemoryLength, + InMemoryBytes, + Overflow), + ?FORALL(O, ?LET(Ops, log_gen(256), expand(Ops, Config)), + collect({log_size, length(O)}, + snapshots_prop(Config, O))) + end) + end, [], 2500). + +single_active(_Config) -> + Size = 2000, + run_proper( + fun () -> + ?FORALL({Length, Bytes, DeliveryLimit, InMemoryLength, InMemoryBytes}, + frequency([{10, {0, 0, 0, 0, 0}}, + {5, {oneof([range(1, 10), undefined]), + oneof([range(1, 1000), undefined]), + oneof([range(1, 3), undefined]), + oneof([range(1, 10), undefined]), + oneof([range(1, 1000), undefined]) + }}]), + begin + Config = config(?FUNCTION_NAME, + Length, + Bytes, + true, + DeliveryLimit, + InMemoryLength, + InMemoryBytes), + ?FORALL(O, ?LET(Ops, log_gen(Size), expand(Ops, Config)), + collect({log_size, length(O)}, + single_active_prop(Config, O, false))) + end) + end, [], Size). + +single_active_ordering(_Config) -> + Size = 2000, + Fun = {-1, fun ({Prev, _}) -> {Prev + 1, Prev + 1} end}, + run_proper( + fun () -> + ?FORALL(O, ?LET(Ops, log_gen_ordered(Size), expand(Ops, Fun)), + collect({log_size, length(O)}, + single_active_prop(config(?FUNCTION_NAME, + undefined, + undefined, + true, + undefined, + undefined, + undefined), O, + true))) + end, [], Size). + +single_active_ordering_01(_Config) -> +% [{enqueue,<0.145.0>,1,0}, +% {enqueue,<0.145.0>,1,1}, +% {checkout,{<<>>,<0.148.0>},{auto,1,simple_prefetch},#{ack => true,args => [],prefetch => 1,username => <<117,115,101,114>>}} +% {enqueue,<0.140.0>,1,2}, +% {settle,{<<>>,<0.148.0>},[0]}] + C1Pid = test_util:fake_pid(node()), + C1 = {<<0>>, C1Pid}, + E = test_util:fake_pid(rabbit@fake_node2), + E2 = test_util:fake_pid(rabbit@fake_node2), + Commands = [ + make_enqueue(E, 1, 0), + make_enqueue(E, 2, 1), + make_checkout(C1, {auto,2,simple_prefetch}), + make_enqueue(E2, 1, 2), + make_settle(C1, [0]) + ], + Conf = config(?FUNCTION_NAME, 0, 0, true, 0, 0, 0), + ?assert(single_active_prop(Conf, Commands, true)), + ok. + +single_active_ordering_02(_Config) -> + %% this results in the pending enqueue being enqueued and violating + %% ordering +% [{checkout, % {<<>>,<0.177.0>}, % {auto,1,simple_prefetch}, +% {enqueue,<0.172.0>,2,1}, +% {down,<0.172.0>,noproc}, +% {settle,{<<>>,<0.177.0>},[0]}] + C1Pid = test_util:fake_pid(node()), + C1 = {<<0>>, C1Pid}, + E = test_util:fake_pid(node()), + Commands = [ + make_checkout(C1, {auto,1,simple_prefetch}), + make_enqueue(E, 2, 1), + %% CANNOT HAPPEN + {down,E,noproc}, + make_settle(C1, [0]) + ], + Conf = config(?FUNCTION_NAME, 0, 0, true, 0, 0, 0), + ?assert(single_active_prop(Conf, Commands, true)), + ok. + +single_active_ordering_03(_Config) -> + C1Pid = test_util:fake_pid(node()), + C1 = {<<1>>, C1Pid}, + C2Pid = test_util:fake_pid(rabbit@fake_node2), + C2 = {<<2>>, C2Pid}, + E = test_util:fake_pid(rabbit@fake_node2), + Commands = [ + make_enqueue(E, 1, 0), + make_enqueue(E, 2, 1), + make_enqueue(E, 3, 2), + make_checkout(C1, {auto,1,simple_prefetch}), + make_checkout(C2, {auto,1,simple_prefetch}), + make_settle(C1, [0]), + make_checkout(C1, cancel), + {down, C1Pid, noconnection} + ], + Conf0 = config(?FUNCTION_NAME, 0, 0, true, 0, 0, 0), + Conf = Conf0#{release_cursor_interval => 100}, + Indexes = lists:seq(1, length(Commands)), + Entries = lists:zip(Indexes, Commands), + try run_log(test_init(Conf), Entries) of + {State, Effects} -> + ct:pal("Effects: ~p~n", [Effects]), + ct:pal("State: ~p~n", [State]), + %% assert C1 has no messages + ?assertNotMatch(#{C1 := _}, State#rabbit_fifo.consumers), + true; + _ -> + true + catch + Err -> + ct:pal("Commands: ~p~nConf~p~n", [Commands, Conf]), + ct:pal("Err: ~p~n", [Err]), + false + end. + +in_memory_limit(_Config) -> + Size = 2000, + run_proper( + fun () -> + ?FORALL({Length, Bytes, SingleActiveConsumer, DeliveryLimit, + InMemoryLength, InMemoryBytes}, + frequency([{10, {0, 0, false, 0, 0, 0}}, + {5, {oneof([range(1, 10), undefined]), + oneof([range(1, 1000), undefined]), + boolean(), + oneof([range(1, 3), undefined]), + range(1, 10), + range(1, 1000) + }}]), + begin + Config = config(?FUNCTION_NAME, + Length, + Bytes, + SingleActiveConsumer, + DeliveryLimit, + InMemoryLength, + InMemoryBytes), + ?FORALL(O, ?LET(Ops, log_gen(Size), expand(Ops, Config)), + collect({log_size, length(O)}, + in_memory_limit_prop(Config, O))) + end) + end, [], Size). + +max_length(_Config) -> + %% tests that max length is never transgressed + Size = 1000, + run_proper( + fun () -> + ?FORALL({Length, SingleActiveConsumer, DeliveryLimit, + InMemoryLength}, + {oneof([range(1, 100), undefined]), + boolean(), + range(1, 3), + range(1, 10) + }, + begin + Config = config(?FUNCTION_NAME, + Length, + undefined, + SingleActiveConsumer, + DeliveryLimit, + InMemoryLength, + undefined), + ?FORALL(O, ?LET(Ops, log_gen_config(Size), + expand(Ops, Config)), + collect({log_size, length(O)}, + max_length_prop(Config, O))) + end) + end, [], Size). + +config(Name, Length, Bytes, SingleActive, DeliveryLimit, + InMemoryLength, InMemoryBytes) -> +config(Name, Length, Bytes, SingleActive, DeliveryLimit, + InMemoryLength, InMemoryBytes, drop_head). + +config(Name, Length, Bytes, SingleActive, DeliveryLimit, + InMemoryLength, InMemoryBytes, Overflow) -> + #{name => Name, + max_length => map_max(Length), + max_bytes => map_max(Bytes), + dead_letter_handler => {?MODULE, banana, []}, + single_active_consumer_on => SingleActive, + delivery_limit => map_max(DeliveryLimit), + max_in_memory_length => map_max(InMemoryLength), + max_in_memory_bytes => map_max(InMemoryBytes), + overflow_strategy => Overflow}. + +map_max(0) -> undefined; +map_max(N) -> N. + +in_memory_limit_prop(Conf0, Commands) -> + Conf = Conf0#{release_cursor_interval => 100}, + Indexes = lists:seq(1, length(Commands)), + Entries = lists:zip(Indexes, Commands), + try run_log(test_init(Conf), Entries) of + {_State, Effects} -> + %% validate message ordering + lists:foldl(fun ({log, Idxs, _}, ReleaseCursorIdx) -> + validate_idx_order(Idxs, ReleaseCursorIdx), + ReleaseCursorIdx; + ({release_cursor, Idx, _}, _) -> + Idx; + (_, Acc) -> + Acc + end, 0, Effects), + true; + _ -> + true + catch + Err -> + ct:pal("Commands: ~p~nConf~p~n", [Commands, Conf]), + ct:pal("Err: ~p~n", [Err]), + false + end. + +max_length_prop(Conf0, Commands) -> + Conf = Conf0#{release_cursor_interval => 100}, + Indexes = lists:seq(1, length(Commands)), + Entries = lists:zip(Indexes, Commands), + Invariant = fun (#rabbit_fifo{cfg = #cfg{max_length = MaxLen}} = S) -> + #{num_ready_messages := MsgReady} = rabbit_fifo:overview(S), + % ct:pal("msg Ready ~w ~w", [MsgReady, MaxLen]), + MsgReady =< MaxLen + end, + try run_log(test_init(Conf), Entries, Invariant) of + {_State, _Effects} -> + true; + _ -> + true + catch + Err -> + ct:pal("Commands: ~p~nConf~p~n", [Commands, Conf]), + ct:pal("Err: ~p~n", [Err]), + false + end. + +validate_idx_order([], _ReleaseCursorIdx) -> + true; +validate_idx_order(Idxs, ReleaseCursorIdx) -> + Min = lists:min(Idxs), + case Min < ReleaseCursorIdx of + true -> + throw({invalid_log_index, Min, ReleaseCursorIdx}); + false -> + ok + end. + +single_active_prop(Conf0, Commands, ValidateOrder) -> + Conf = Conf0#{release_cursor_interval => 100}, + Indexes = lists:seq(1, length(Commands)), + Entries = lists:zip(Indexes, Commands), + %% invariant: there can only be one active consumer at any one time + %% there can however be multiple cancelled consumers + Invariant = fun (#rabbit_fifo{consumers = Consumers}) -> + Up = maps:filter(fun (_, #consumer{status = S}) -> + S == up + end, Consumers), + map_size(Up) =< 1 + end, + try run_log(test_init(Conf), Entries, Invariant) of + {_State, Effects} when ValidateOrder -> + % ct:pal("Effects: ~p~n", [Effects]), + % ct:pal("State: ~p~n", [State]), + %% validate message ordering + lists:foldl(fun ({send_msg, Pid, {delivery, Tag, Msgs}, ra_event}, + Acc) -> + validate_msg_order({Tag, Pid}, Msgs, Acc); + (_, Acc) -> + Acc + end, -1, Effects), + true; + _ -> + true + catch + Err -> + ct:pal("Commands: ~p~nConf~p~n", [Commands, Conf]), + ct:pal("Err: ~p~n", [Err]), + false + end. + +%% single active consumer ordering invariant: +%% only redelivered messages can go backwards +validate_msg_order(_, [], S) -> + S; +validate_msg_order(Cid, [{_, {H, Num}} | Rem], PrevMax) -> + Redelivered = is_map(H) andalso maps:is_key(delivery_count, H), + case undefined of + _ when Num == PrevMax + 1 -> + %% forwards case + validate_msg_order(Cid, Rem, Num); + _ when Redelivered andalso Num =< PrevMax -> + %% the seq is lower but this is a redelivery + %% when the consumer changed and the next messages has been redelivered + %% we may go backwards but keep the highest seen + validate_msg_order(Cid, Rem, PrevMax); + _ -> + ct:pal("out of order ~w Prev ~w Curr ~w Redel ~w", + [Cid, PrevMax, Num, Redelivered]), + throw({outoforder, Cid, PrevMax, Num}) + end. + + + + +dump_generated(Conf, Commands) -> + ct:pal("Commands: ~p~nConf~p~n", [Commands, Conf]), + true. + +snapshots_prop(Conf, Commands) -> + try run_snapshot_test(Conf, Commands) of + _ -> true + catch + Err -> + ct:pal("Commands: ~p~nConf~p~n", [Commands, Conf]), + ct:pal("Err: ~p~n", [Err]), + false + end. + +log_gen(Size) -> + log_gen(Size, binary()). + +log_gen(Size, _Body) -> + Nodes = [node(), + fakenode@fake, + fakenode@fake2 + ], + ?LET(EPids, vector(2, pid_gen(Nodes)), + ?LET(CPids, vector(2, pid_gen(Nodes)), + resize(Size, + list( + frequency( + [{20, enqueue_gen(oneof(EPids))}, + {40, {input_event, + frequency([{10, settle}, + {2, return}, + {2, discard}, + {2, requeue}])}}, + {2, checkout_gen(oneof(CPids))}, + {1, checkout_cancel_gen(oneof(CPids))}, + {1, down_gen(oneof(EPids ++ CPids))}, + {1, nodeup_gen(Nodes)}, + {1, purge} + ]))))). + +log_gen_config(Size) -> + Nodes = [node(), + fakenode@fake, + fakenode@fake2 + ], + ?LET(EPids, vector(2, pid_gen(Nodes)), + ?LET(CPids, vector(2, pid_gen(Nodes)), + resize(Size, + list( + frequency( + [{20, enqueue_gen(oneof(EPids))}, + {40, {input_event, + frequency([{5, settle}, + {5, return}, + {2, discard}, + {2, requeue}])}}, + {2, checkout_gen(oneof(CPids))}, + {1, checkout_cancel_gen(oneof(CPids))}, + {1, down_gen(oneof(EPids ++ CPids))}, + {1, nodeup_gen(Nodes)}, + {1, purge}, + {1, ?LET({MaxInMem, + MaxLen}, + {choose(1, 10), + choose(1, 10)}, + {update_config, + #{max_in_memory_length => MaxInMem, + max_length => MaxLen}}) + }]))))). + +log_gen_ordered(Size) -> + Nodes = [node(), + fakenode@fake, + fakenode@fake2 + ], + ?LET(EPids, vector(1, pid_gen(Nodes)), + ?LET(CPids, vector(8, pid_gen(Nodes)), + resize(Size, + list( + frequency( + [{20, enqueue_gen(oneof(EPids), 10, 0)}, + {40, {input_event, + frequency([{15, settle}, + {1, return}, + {1, discard}, + {1, requeue}])}}, + {7, checkout_gen(oneof(CPids))}, + {2, checkout_cancel_gen(oneof(CPids))}, + {2, down_gen(oneof(EPids ++ CPids))}, + {1, nodeup_gen(Nodes)} + ]))))). + +monotonic_gen() -> + ?LET(_, integer(), erlang:unique_integer([positive, monotonic])). + +pid_gen(Nodes) -> + ?LET(Node, oneof(Nodes), + test_util:fake_pid(atom_to_binary(Node, utf8))). + +down_gen(Pid) -> + ?LET(E, {down, Pid, oneof([noconnection, noproc])}, E). + +nodeup_gen(Nodes) -> + {nodeup, oneof(Nodes)}. + +enqueue_gen(Pid) -> + enqueue_gen(Pid, 10, 1). + +enqueue_gen(Pid, Enq, Del) -> + ?LET(E, {enqueue, Pid, + frequency([{Enq, enqueue}, + {Del, delay}]), + binary()}, E). + +checkout_cancel_gen(Pid) -> + {checkout, Pid, cancel}. + +checkout_gen(Pid) -> + %% pid, tag, prefetch + ?LET(C, {checkout, {binary(), Pid}, choose(1, 100)}, C). + + +-record(t, {state = rabbit_fifo:init(#{name => proper, + queue_resource => blah, + release_cursor_interval => 1}) + :: rabbit_fifo:state(), + index = 1 :: non_neg_integer(), %% raft index + enqueuers = #{} :: #{pid() => term()}, + consumers = #{} :: #{{binary(), pid()} => term()}, + effects = queue:new() :: queue:queue(), + %% to transform the body + enq_body_fun = {0, fun ra_lib:id/1}, + config :: map(), + log = [] :: list(), + down = #{} :: #{pid() => noproc | noconnection} + }). + +expand(Ops, Config) -> + expand(Ops, Config, {undefined, fun ra_lib:id/1}). + +expand(Ops, Config, EnqFun) -> + %% execute each command against a rabbit_fifo state and capture all relevant + %% effects + T = #t{enq_body_fun = EnqFun, + config = Config}, + #t{effects = Effs} = T1 = lists:foldl(fun handle_op/2, T, Ops), + %% process the remaining effect + #t{log = Log} = lists:foldl(fun do_apply/2, + T1#t{effects = queue:new()}, + queue:to_list(Effs)), + + lists:reverse(Log). + + +handle_op({enqueue, Pid, When, Data}, + #t{enqueuers = Enqs0, + enq_body_fun = {EnqSt0, Fun}, + down = Down, + effects = Effs} = T) -> + case Down of + #{Pid := noproc} -> + %% if it's a noproc then it cannot exist - can it? + %% drop operation + T; + _ -> + Enqs = maps:update_with(Pid, fun (Seq) -> Seq + 1 end, 1, Enqs0), + MsgSeq = maps:get(Pid, Enqs), + {EnqSt, Msg} = Fun({EnqSt0, Data}), + Cmd = rabbit_fifo:make_enqueue(Pid, MsgSeq, Msg), + case When of + enqueue -> + do_apply(Cmd, T#t{enqueuers = Enqs, + enq_body_fun = {EnqSt, Fun}}); + delay -> + %% just put the command on the effects queue + T#t{effects = queue:in(Cmd, Effs), + enqueuers = Enqs, + enq_body_fun = {EnqSt, Fun}} + end + end; +handle_op({checkout, Pid, cancel}, #t{consumers = Cons0} = T) -> + case maps:keys( + maps:filter(fun ({_, P}, _) when P == Pid -> true; + (_, _) -> false + end, Cons0)) of + [CId | _] -> + Cons = maps:remove(CId, Cons0), + Cmd = rabbit_fifo:make_checkout(CId, cancel, #{}), + do_apply(Cmd, T#t{consumers = Cons}); + _ -> + T + end; +handle_op({checkout, CId, Prefetch}, #t{consumers = Cons0} = T) -> + case Cons0 of + #{CId := _} -> + %% ignore if it already exists + T; + _ -> + Cons = maps:put(CId, ok, Cons0), + Cmd = rabbit_fifo:make_checkout(CId, + {auto, Prefetch, simple_prefetch}, + #{ack => true, + prefetch => Prefetch, + username => <<"user">>, + args => []}), + + do_apply(Cmd, T#t{consumers = Cons}) + end; +handle_op({down, Pid, Reason} = Cmd, #t{down = Down} = T) -> + case Down of + #{Pid := noproc} -> + %% it it permanently down, cannot upgrade + T; + _ -> + %% it is either not down or down with noconnection + do_apply(Cmd, T#t{down = maps:put(Pid, Reason, Down)}) + end; +handle_op({nodeup, _} = Cmd, T) -> + do_apply(Cmd, T); +handle_op({input_event, requeue}, #t{effects = Effs} = T) -> + %% this simulates certain settlements arriving out of order + case queue:out(Effs) of + {{value, Cmd}, Q} -> + T#t{effects = queue:in(Cmd, Q)}; + _ -> + T + end; +handle_op({input_event, Settlement}, #t{effects = Effs, + down = Down} = T) -> + case queue:out(Effs) of + {{value, {settle, MsgIds, CId}}, Q} -> + Cmd = case Settlement of + settle -> rabbit_fifo:make_settle(CId, MsgIds); + return -> rabbit_fifo:make_return(CId, MsgIds); + discard -> rabbit_fifo:make_discard(CId, MsgIds) + end, + do_apply(Cmd, T#t{effects = Q}); + {{value, {enqueue, Pid, _, _} = Cmd}, Q} -> + case maps:is_key(Pid, Down) of + true -> + %% enqueues cannot arrive after down for the same process + %% drop message + T#t{effects = Q}; + false -> + do_apply(Cmd, T#t{effects = Q}) + end; + _ -> + T + end; +handle_op(purge, T) -> + do_apply(rabbit_fifo:make_purge(), T); +handle_op({update_config, Changes}, #t{config = Conf} = T) -> + Config = maps:merge(Conf, Changes), + do_apply(rabbit_fifo:make_update_config(Config), T). + + +do_apply(Cmd, #t{effects = Effs, + index = Index, state = S0, + down = Down, + log = Log} = T) -> + case Cmd of + {enqueue, Pid, _, _} when is_map_key(Pid, Down) -> + %% down + T; + _ -> + {St, Effects} = case rabbit_fifo:apply(meta(Index), Cmd, S0) of + {S, _, E} when is_list(E) -> + {S, E}; + {S, _, E} -> + {S, [E]}; + {S, _} -> + {S, []} + end, + + T#t{state = St, + index = Index + 1, + effects = enq_effs(Effects, Effs), + log = [Cmd | Log]} + end. + +enq_effs([], Q) -> Q; +enq_effs([{send_msg, P, {delivery, CTag, Msgs}, ra_event} | Rem], Q) -> + MsgIds = [I || {I, _} <- Msgs], + %% always make settle commands by default + %% they can be changed depending on the input event later + Cmd = rabbit_fifo:make_settle({CTag, P}, MsgIds), + enq_effs(Rem, queue:in(Cmd, Q)); +enq_effs([_ | Rem], Q) -> + enq_effs(Rem, Q). + + +%% Utility +run_proper(Fun, Args, NumTests) -> + ?assertEqual( + true, + proper:counterexample( + erlang:apply(Fun, Args), + [{numtests, NumTests}, + {on_output, fun(".", _) -> ok; % don't print the '.'s on new lines + (F, A) -> ct:pal(?LOW_IMPORTANCE, F, A) + end}])). + +run_snapshot_test(Conf, Commands) -> + %% create every incremental permutation of the commands lists + %% and run the snapshot tests against that + ct:pal("running snapshot test with ~b commands using config ~p", + [length(Commands), Conf]), + [begin + % ?debugFmt("~w running command to ~w~n", [?FUNCTION_NAME, lists:last(C)]), + run_snapshot_test0(Conf, C) + end || C <- prefixes(Commands, 1, [])]. + +run_snapshot_test0(Conf, Commands) -> + Indexes = lists:seq(1, length(Commands)), + Entries = lists:zip(Indexes, Commands), + {State0, Effects} = run_log(test_init(Conf), Entries), + State = rabbit_fifo:normalize(State0), + + [begin + % ct:pal("release_cursor: ~b~n", [SnapIdx]), + %% drop all entries below and including the snapshot + Filtered = lists:dropwhile(fun({X, _}) when X =< SnapIdx -> true; + (_) -> false + end, Entries), + {S0, _} = run_log(SnapState, Filtered), + S = rabbit_fifo:normalize(S0), + % assert log can be restored from any release cursor index + case S of + State -> ok; + _ -> + ct:pal("Snapshot tests failed run log:~n" + "~p~n from ~n~p~n Entries~n~p~n" + "Config: ~p~n", + [Filtered, SnapState, Entries, Conf]), + ct:pal("Expected~n~p~nGot:~n~p", [State, S]), + ?assertEqual(State, S) + end + end || {release_cursor, SnapIdx, SnapState} <- Effects], + ok. + +%% transforms [1,2,3] into [[1,2,3], [1,2], [1]] +prefixes(Source, N, Acc) when N > length(Source) -> + lists:reverse(Acc); +prefixes(Source, N, Acc) -> + {X, _} = lists:split(N, Source), + prefixes(Source, N+1, [X | Acc]). + +run_log(InitState, Entries) -> + run_log(InitState, Entries, fun(_) -> true end). + +run_log(InitState, Entries, InvariantFun) -> + Invariant = fun(E, S) -> + case InvariantFun(S) of + true -> ok; + false -> + throw({invariant, E, S}) + end + end, + + lists:foldl(fun ({Idx, E}, {Acc0, Efx0}) -> + case rabbit_fifo:apply(meta(Idx), E, Acc0) of + {Acc, _, Efx} when is_list(Efx) -> + Invariant(E, Acc), + {Acc, Efx0 ++ Efx}; + {Acc, _, Efx} -> + Invariant(E, Acc), + {Acc, Efx0 ++ [Efx]}; + {Acc, _} -> + Invariant(E, Acc), + {Acc, Efx0} + end + end, {InitState, []}, Entries). + +test_init(Conf) -> + Default = #{queue_resource => blah, + release_cursor_interval => 0, + metrics_handler => {?MODULE, metrics_handler, []}}, + rabbit_fifo:init(maps:merge(Default, Conf)). + +meta(Idx) -> + #{index => Idx, term => 1, system_time => 0}. + +make_checkout(Cid, Spec) -> + rabbit_fifo:make_checkout(Cid, Spec, #{}). + +make_enqueue(Pid, Seq, Msg) -> + rabbit_fifo:make_enqueue(Pid, Seq, Msg). + +make_settle(Cid, MsgIds) -> + rabbit_fifo:make_settle(Cid, MsgIds). + +make_return(Cid, MsgIds) -> + rabbit_fifo:make_return(Cid, MsgIds). diff --git a/deps/rabbit/test/rabbit_fifo_v0_SUITE.erl b/deps/rabbit/test/rabbit_fifo_v0_SUITE.erl new file mode 100644 index 0000000000..fcb84377de --- /dev/null +++ b/deps/rabbit/test/rabbit_fifo_v0_SUITE.erl @@ -0,0 +1,1392 @@ +-module(rabbit_fifo_v0_SUITE). + +%% rabbit_fifo unit tests suite + +-compile(export_all). + +-compile({no_auto_import, [apply/3]}). +-export([ + ]). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("rabbit_common/include/rabbit.hrl"). +-include("src/rabbit_fifo_v0.hrl"). + +%%%=================================================================== +%%% Common Test callbacks +%%%=================================================================== + +all() -> + [ + {group, tests} + ]. + + +%% replicate eunit like test resultion +all_tests() -> + [F || {F, _} <- ?MODULE:module_info(functions), + re:run(atom_to_list(F), "_test$") /= nomatch]. + +groups() -> + [ + {tests, [], all_tests()} + ]. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_Group, Config) -> + Config. + +end_per_group(_Group, _Config) -> + ok. + +init_per_testcase(_TestCase, Config) -> + Config. + +end_per_testcase(_TestCase, _Config) -> + ok. + +%%%=================================================================== +%%% Test cases +%%%=================================================================== + +-define(ASSERT_EFF(EfxPat, Effects), + ?ASSERT_EFF(EfxPat, true, Effects)). + +-define(ASSERT_EFF(EfxPat, Guard, Effects), + ?assert(lists:any(fun (EfxPat) when Guard -> true; + (_) -> false + end, Effects))). + +-define(ASSERT_NO_EFF(EfxPat, Effects), + ?assert(not lists:any(fun (EfxPat) -> true; + (_) -> false + end, Effects))). + +-define(assertNoEffect(EfxPat, Effects), + ?assert(not lists:any(fun (EfxPat) -> true; + (_) -> false + end, Effects))). + +test_init(Name) -> + init(#{name => Name, + queue_resource => rabbit_misc:r("/", queue, + atom_to_binary(Name, utf8)), + release_cursor_interval => 0}). + +enq_enq_checkout_test(_) -> + Cid = {<<"enq_enq_checkout_test">>, self()}, + {State1, _} = enq(1, 1, first, test_init(test)), + {State2, _} = enq(2, 2, second, State1), + {_State3, _, Effects} = + apply(meta(3), + rabbit_fifo_v0:make_checkout(Cid, {once, 2, simple_prefetch}, #{}), + State2), + ?ASSERT_EFF({monitor, _, _}, Effects), + ?ASSERT_EFF({send_msg, _, {delivery, _, _}, _}, Effects), + ok. + +credit_enq_enq_checkout_settled_credit_test(_) -> + Cid = {?FUNCTION_NAME, self()}, + {State1, _} = enq(1, 1, first, test_init(test)), + {State2, _} = enq(2, 2, second, State1), + {State3, _, Effects} = + apply(meta(3), rabbit_fifo_v0:make_checkout(Cid, {auto, 1, credited}, #{}), State2), + ?ASSERT_EFF({monitor, _, _}, Effects), + Deliveries = lists:filter(fun ({send_msg, _, {delivery, _, _}, _}) -> true; + (_) -> false + end, Effects), + ?assertEqual(1, length(Deliveries)), + %% settle the delivery this should _not_ result in further messages being + %% delivered + {State4, SettledEffects} = settle(Cid, 4, 1, State3), + ?assertEqual(false, lists:any(fun ({send_msg, _, {delivery, _, _}, _}) -> + true; + (_) -> false + end, SettledEffects)), + %% granting credit (3) should deliver the second msg if the receivers + %% delivery count is (1) + {State5, CreditEffects} = credit(Cid, 5, 1, 1, false, State4), + % ?debugFmt("CreditEffects ~p ~n~p", [CreditEffects, State4]), + ?ASSERT_EFF({send_msg, _, {delivery, _, _}, _}, CreditEffects), + {_State6, FinalEffects} = enq(6, 3, third, State5), + ?assertEqual(false, lists:any(fun ({send_msg, _, {delivery, _, _}, _}) -> + true; + (_) -> false + end, FinalEffects)), + ok. + +credit_with_drained_test(_) -> + Cid = {?FUNCTION_NAME, self()}, + State0 = test_init(test), + %% checkout with a single credit + {State1, _, _} = + apply(meta(1), rabbit_fifo_v0:make_checkout(Cid, {auto, 1, credited},#{}), + State0), + ?assertMatch(#?STATE{consumers = #{Cid := #consumer{credit = 1, + delivery_count = 0}}}, + State1), + {State, Result, _} = + apply(meta(3), rabbit_fifo_v0:make_credit(Cid, 0, 5, true), State1), + ?assertMatch(#?STATE{consumers = #{Cid := #consumer{credit = 0, + delivery_count = 5}}}, + State), + ?assertEqual({multi, [{send_credit_reply, 0}, + {send_drained, {?FUNCTION_NAME, 5}}]}, + Result), + ok. + +credit_and_drain_test(_) -> + Cid = {?FUNCTION_NAME, self()}, + {State1, _} = enq(1, 1, first, test_init(test)), + {State2, _} = enq(2, 2, second, State1), + %% checkout without any initial credit (like AMQP 1.0 would) + {State3, _, CheckEffs} = + apply(meta(3), rabbit_fifo_v0:make_checkout(Cid, {auto, 0, credited}, #{}), + State2), + + ?ASSERT_NO_EFF({send_msg, _, {delivery, _, _}}, CheckEffs), + {State4, {multi, [{send_credit_reply, 0}, + {send_drained, {?FUNCTION_NAME, 2}}]}, + Effects} = apply(meta(4), rabbit_fifo_v0:make_credit(Cid, 4, 0, true), State3), + ?assertMatch(#?STATE{consumers = #{Cid := #consumer{credit = 0, + delivery_count = 4}}}, + State4), + + ?ASSERT_EFF({send_msg, _, {delivery, _, [{_, {_, first}}, + {_, {_, second}}]}, _}, Effects), + {_State5, EnqEffs} = enq(5, 2, third, State4), + ?ASSERT_NO_EFF({send_msg, _, {delivery, _, _}}, EnqEffs), + ok. + + + +enq_enq_deq_test(_) -> + Cid = {?FUNCTION_NAME, self()}, + {State1, _} = enq(1, 1, first, test_init(test)), + {State2, _} = enq(2, 2, second, State1), + % get returns a reply value + NumReady = 1, + {_State3, {dequeue, {0, {_, first}}, NumReady}, [{monitor, _, _}]} = + apply(meta(3), rabbit_fifo_v0:make_checkout(Cid, {dequeue, unsettled}, #{}), + State2), + ok. + +enq_enq_deq_deq_settle_test(_) -> + Cid = {?FUNCTION_NAME, self()}, + {State1, _} = enq(1, 1, first, test_init(test)), + {State2, _} = enq(2, 2, second, State1), + % get returns a reply value + {State3, {dequeue, {0, {_, first}}, 1}, [{monitor, _, _}]} = + apply(meta(3), rabbit_fifo_v0:make_checkout(Cid, {dequeue, unsettled}, #{}), + State2), + {_State4, {dequeue, empty}} = + apply(meta(4), rabbit_fifo_v0:make_checkout(Cid, {dequeue, unsettled}, #{}), + State3), + ok. + +enq_enq_checkout_get_settled_test(_) -> + Cid = {?FUNCTION_NAME, self()}, + {State1, _} = enq(1, 1, first, test_init(test)), + % get returns a reply value + {_State2, {dequeue, {0, {_, first}}, _}, _Effs} = + apply(meta(3), rabbit_fifo_v0:make_checkout(Cid, {dequeue, settled}, #{}), + State1), + ok. + +checkout_get_empty_test(_) -> + Cid = {?FUNCTION_NAME, self()}, + State = test_init(test), + {_State2, {dequeue, empty}} = + apply(meta(1), rabbit_fifo_v0:make_checkout(Cid, {dequeue, unsettled}, #{}), State), + ok. + +untracked_enq_deq_test(_) -> + Cid = {?FUNCTION_NAME, self()}, + State0 = test_init(test), + {State1, _, _} = apply(meta(1), + rabbit_fifo_v0:make_enqueue(undefined, undefined, first), + State0), + {_State2, {dequeue, {0, {_, first}}, _}, _} = + apply(meta(3), rabbit_fifo_v0:make_checkout(Cid, {dequeue, settled}, #{}), State1), + ok. + +release_cursor_test(_) -> + Cid = {?FUNCTION_NAME, self()}, + {State1, _} = enq(1, 1, first, test_init(test)), + {State2, _} = enq(2, 2, second, State1), + {State3, _} = check(Cid, 3, 10, State2), + % no release cursor effect at this point + {State4, _} = settle(Cid, 4, 1, State3), + {_Final, Effects1} = settle(Cid, 5, 0, State4), + % empty queue forwards release cursor all the way + ?ASSERT_EFF({release_cursor, 5, _}, Effects1), + ok. + +checkout_enq_settle_test(_) -> + Cid = {?FUNCTION_NAME, self()}, + {State1, [{monitor, _, _} | _]} = check(Cid, 1, test_init(test)), + {State2, Effects0} = enq(2, 1, first, State1), + ?ASSERT_EFF({send_msg, _, + {delivery, ?FUNCTION_NAME, + [{0, {_, first}}]}, _}, + Effects0), + {State3, [_Inactive]} = enq(3, 2, second, State2), + {_, _Effects} = settle(Cid, 4, 0, State3), + % the release cursor is the smallest raft index that does not + % contribute to the state of the application + % ?ASSERT_EFF({release_cursor, 2, _}, Effects), + ok. + +out_of_order_enqueue_test(_) -> + Cid = {?FUNCTION_NAME, self()}, + {State1, [{monitor, _, _} | _]} = check_n(Cid, 5, 5, test_init(test)), + {State2, Effects2} = enq(2, 1, first, State1), + ?ASSERT_EFF({send_msg, _, {delivery, _, [{_, {_, first}}]}, _}, Effects2), + % assert monitor was set up + ?ASSERT_EFF({monitor, _, _}, Effects2), + % enqueue seq num 3 and 4 before 2 + {State3, Effects3} = enq(3, 3, third, State2), + ?assertNoEffect({send_msg, _, {delivery, _, _}, _}, Effects3), + {State4, Effects4} = enq(4, 4, fourth, State3), + % assert no further deliveries where made + ?assertNoEffect({send_msg, _, {delivery, _, _}, _}, Effects4), + {_State5, Effects5} = enq(5, 2, second, State4), + % assert two deliveries were now made + ?ASSERT_EFF({send_msg, _, {delivery, _, [{_, {_, second}}, + {_, {_, third}}, + {_, {_, fourth}}]}, _}, + Effects5), + ok. + +out_of_order_first_enqueue_test(_) -> + Cid = {?FUNCTION_NAME, self()}, + {State1, _} = check_n(Cid, 5, 5, test_init(test)), + {_State2, Effects2} = enq(2, 10, first, State1), + ?ASSERT_EFF({monitor, process, _}, Effects2), + ?assertNoEffect({send_msg, _, {delivery, _, [{_, {_, first}}]}, _}, + Effects2), + ok. + +duplicate_enqueue_test(_) -> + Cid = {<<"duplicate_enqueue_test">>, self()}, + {State1, [{monitor, _, _} | _]} = check_n(Cid, 5, 5, test_init(test)), + {State2, Effects2} = enq(2, 1, first, State1), + ?ASSERT_EFF({send_msg, _, {delivery, _, [{_, {_, first}}]}, _}, Effects2), + {_State3, Effects3} = enq(3, 1, first, State2), + ?assertNoEffect({send_msg, _, {delivery, _, [{_, {_, first}}]}, _}, Effects3), + ok. + +return_test(_) -> + Cid = {<<"cid">>, self()}, + Cid2 = {<<"cid2">>, self()}, + {State0, _} = enq(1, 1, msg, test_init(test)), + {State1, _} = check_auto(Cid, 2, State0), + {State2, _} = check_auto(Cid2, 3, State1), + {State3, _, _} = apply(meta(4), rabbit_fifo_v0:make_return(Cid, [0]), State2), + ?assertMatch(#{Cid := #consumer{checked_out = C}} when map_size(C) == 0, + State3#?STATE.consumers), + ?assertMatch(#{Cid2 := #consumer{checked_out = C2}} when map_size(C2) == 1, + State3#?STATE.consumers), + ok. + +return_dequeue_delivery_limit_test(_) -> + Init = init(#{name => test, + queue_resource => rabbit_misc:r("/", queue, + atom_to_binary(test, utf8)), + release_cursor_interval => 0, + delivery_limit => 1}), + {State0, _} = enq(1, 1, msg, Init), + + Cid = {<<"cid">>, self()}, + Cid2 = {<<"cid2">>, self()}, + + {State1, {MsgId1, _}} = deq(2, Cid, unsettled, State0), + {State2, _, _} = apply(meta(4), rabbit_fifo_v0:make_return(Cid, [MsgId1]), + State1), + + {State3, {MsgId2, _}} = deq(2, Cid2, unsettled, State2), + {State4, _, _} = apply(meta(4), rabbit_fifo_v0:make_return(Cid2, [MsgId2]), + State3), + ?assertMatch(#{num_messages := 0}, rabbit_fifo_v0:overview(State4)), + ok. + +return_non_existent_test(_) -> + Cid = {<<"cid">>, self()}, + {State0, [_, _Inactive]} = enq(1, 1, second, test_init(test)), + % return non-existent + {_State2, _} = apply(meta(3), rabbit_fifo_v0:make_return(Cid, [99]), State0), + ok. + +return_checked_out_test(_) -> + Cid = {<<"cid">>, self()}, + {State0, [_, _]} = enq(1, 1, first, test_init(test)), + {State1, [_Monitor, + {send_msg, _, {delivery, _, [{MsgId, _}]}, _}, + {aux, active} | _ ]} = check_auto(Cid, 2, State0), + % returning immediately checks out the same message again + {_, ok, [{send_msg, _, {delivery, _, [{_, _}]}, _}, + {aux, active}]} = + apply(meta(3), rabbit_fifo_v0:make_return(Cid, [MsgId]), State1), + ok. + +return_checked_out_limit_test(_) -> + Cid = {<<"cid">>, self()}, + Init = init(#{name => test, + queue_resource => rabbit_misc:r("/", queue, + atom_to_binary(test, utf8)), + release_cursor_interval => 0, + delivery_limit => 1}), + {State0, [_, _]} = enq(1, 1, first, Init), + {State1, [_Monitor, + {send_msg, _, {delivery, _, [{MsgId, _}]}, _}, + {aux, active} | _ ]} = check_auto(Cid, 2, State0), + % returning immediately checks out the same message again + {State2, ok, [{send_msg, _, {delivery, _, [{MsgId2, _}]}, _}, + {aux, active}]} = + apply(meta(3), rabbit_fifo_v0:make_return(Cid, [MsgId]), State1), + {#?STATE{ra_indexes = RaIdxs}, ok, [_ReleaseEff]} = + apply(meta(4), rabbit_fifo_v0:make_return(Cid, [MsgId2]), State2), + ?assertEqual(0, rabbit_fifo_index:size(RaIdxs)), + ok. + +return_auto_checked_out_test(_) -> + Cid = {<<"cid">>, self()}, + {State00, [_, _]} = enq(1, 1, first, test_init(test)), + {State0, [_]} = enq(2, 2, second, State00), + % it first active then inactive as the consumer took on but cannot take + % any more + {State1, [_Monitor, + {send_msg, _, {delivery, _, [{MsgId, _}]}, _}, + {aux, active}, + {aux, inactive} + ]} = check_auto(Cid, 2, State0), + % return should include another delivery + {_State2, _, Effects} = apply(meta(3), rabbit_fifo_v0:make_return(Cid, [MsgId]), State1), + ?ASSERT_EFF({send_msg, _, + {delivery, _, [{_, {#{delivery_count := 1}, first}}]}, _}, + Effects), + ok. + +cancelled_checkout_out_test(_) -> + Cid = {<<"cid">>, self()}, + {State00, [_, _]} = enq(1, 1, first, test_init(test)), + {State0, [_]} = enq(2, 2, second, State00), + {State1, _} = check_auto(Cid, 2, State0), + % cancelled checkout should not return pending messages to queue + {State2, _, _} = apply(meta(3), rabbit_fifo_v0:make_checkout(Cid, cancel, #{}), State1), + ?assertEqual(1, maps:size(State2#?STATE.messages)), + ?assertEqual(0, lqueue:len(State2#?STATE.returns)), + + {State3, {dequeue, empty}} = + apply(meta(3), rabbit_fifo_v0:make_checkout(Cid, {dequeue, settled}, #{}), State2), + %% settle + {State4, ok, _} = + apply(meta(4), rabbit_fifo_v0:make_settle(Cid, [0]), State3), + + {_State, {dequeue, {_, {_, second}}, _}, _} = + apply(meta(5), rabbit_fifo_v0:make_checkout(Cid, {dequeue, settled}, #{}), State4), + ok. + +down_with_noproc_consumer_returns_unsettled_test(_) -> + Cid = {<<"down_consumer_returns_unsettled_test">>, self()}, + {State0, [_, _]} = enq(1, 1, second, test_init(test)), + {State1, [{monitor, process, Pid} | _]} = check(Cid, 2, State0), + {State2, _, _} = apply(meta(3), {down, Pid, noproc}, State1), + {_State, Effects} = check(Cid, 4, State2), + ?ASSERT_EFF({monitor, process, _}, Effects), + ok. + +down_with_noconnection_marks_suspect_and_node_is_monitored_test(_) -> + Pid = spawn(fun() -> ok end), + Cid = {<<"down_with_noconnect">>, Pid}, + Self = self(), + Node = node(Pid), + {State0, Effects0} = enq(1, 1, second, test_init(test)), + ?ASSERT_EFF({monitor, process, P}, P =:= Self, Effects0), + {State1, Effects1} = check_auto(Cid, 2, State0), + #consumer{credit = 0} = maps:get(Cid, State1#?STATE.consumers), + ?ASSERT_EFF({monitor, process, P}, P =:= Pid, Effects1), + % monitor both enqueuer and consumer + % because we received a noconnection we now need to monitor the node + {State2a, _, _} = apply(meta(3), {down, Pid, noconnection}, State1), + #consumer{credit = 1, + checked_out = Ch, + status = suspected_down} = maps:get(Cid, State2a#?STATE.consumers), + ?assertEqual(#{}, Ch), + %% validate consumer has credit + {State2, _, Effects2} = apply(meta(3), {down, Self, noconnection}, State2a), + ?ASSERT_EFF({monitor, node, _}, Effects2), + ?assertNoEffect({demonitor, process, _}, Effects2), + % when the node comes up we need to retry the process monitors for the + % disconnected processes + {State3, _, Effects3} = apply(meta(3), {nodeup, Node}, State2), + #consumer{status = up} = maps:get(Cid, State3#?STATE.consumers), + % try to re-monitor the suspect processes + ?ASSERT_EFF({monitor, process, P}, P =:= Pid, Effects3), + ?ASSERT_EFF({monitor, process, P}, P =:= Self, Effects3), + ok. + +down_with_noconnection_returns_unack_test(_) -> + Pid = spawn(fun() -> ok end), + Cid = {<<"down_with_noconnect">>, Pid}, + {State0, _} = enq(1, 1, second, test_init(test)), + ?assertEqual(1, maps:size(State0#?STATE.messages)), + ?assertEqual(0, lqueue:len(State0#?STATE.returns)), + {State1, {_, _}} = deq(2, Cid, unsettled, State0), + ?assertEqual(0, maps:size(State1#?STATE.messages)), + ?assertEqual(0, lqueue:len(State1#?STATE.returns)), + {State2a, _, _} = apply(meta(3), {down, Pid, noconnection}, State1), + ?assertEqual(0, maps:size(State2a#?STATE.messages)), + ?assertEqual(1, lqueue:len(State2a#?STATE.returns)), + ?assertMatch(#consumer{checked_out = Ch, + status = suspected_down} + when map_size(Ch) == 0, + maps:get(Cid, State2a#?STATE.consumers)), + ok. + +down_with_noproc_enqueuer_is_cleaned_up_test(_) -> + State00 = test_init(test), + Pid = spawn(fun() -> ok end), + {State0, _, Effects0} = apply(meta(1), rabbit_fifo_v0:make_enqueue(Pid, 1, first), State00), + ?ASSERT_EFF({monitor, process, _}, Effects0), + {State1, _, _} = apply(meta(3), {down, Pid, noproc}, State0), + % ensure there are no enqueuers + ?assert(0 =:= maps:size(State1#?STATE.enqueuers)), + ok. + +discarded_message_without_dead_letter_handler_is_removed_test(_) -> + Cid = {<<"completed_consumer_yields_demonitor_effect_test">>, self()}, + {State0, [_, _]} = enq(1, 1, first, test_init(test)), + {State1, Effects1} = check_n(Cid, 2, 10, State0), + ?ASSERT_EFF({send_msg, _, + {delivery, _, [{0, {_, first}}]}, _}, + Effects1), + {_State2, _, Effects2} = apply(meta(1), + rabbit_fifo_v0:make_discard(Cid, [0]), State1), + ?assertNoEffect({send_msg, _, + {delivery, _, [{0, {_, first}}]}, _}, + Effects2), + ok. + +discarded_message_with_dead_letter_handler_emits_log_effect_test(_) -> + Cid = {<<"completed_consumer_yields_demonitor_effect_test">>, self()}, + State00 = init(#{name => test, + queue_resource => rabbit_misc:r(<<"/">>, queue, <<"test">>), + dead_letter_handler => + {somemod, somefun, [somearg]}}), + {State0, [_, _]} = enq(1, 1, first, State00), + {State1, Effects1} = check_n(Cid, 2, 10, State0), + ?ASSERT_EFF({send_msg, _, + {delivery, _, [{0, {_, first}}]}, _}, + Effects1), + {_State2, _, Effects2} = apply(meta(1), rabbit_fifo_v0:make_discard(Cid, [0]), State1), + % assert mod call effect with appended reason and message + ?ASSERT_EFF({log, _RaftIdxs, _}, Effects2), + ok. + +tick_test(_) -> + Cid = {<<"c">>, self()}, + Cid2 = {<<"c2">>, self()}, + {S0, _} = enq(1, 1, <<"fst">>, test_init(?FUNCTION_NAME)), + {S1, _} = enq(2, 2, <<"snd">>, S0), + {S2, {MsgId, _}} = deq(3, Cid, unsettled, S1), + {S3, {_, _}} = deq(4, Cid2, unsettled, S2), + {S4, _, _} = apply(meta(5), rabbit_fifo_v0:make_return(Cid, [MsgId]), S3), + + [{mod_call, rabbit_quorum_queue, handle_tick, + [#resource{}, + {?FUNCTION_NAME, 1, 1, 2, 1, 3, 3}, + [_Node] + ]}] = rabbit_fifo_v0:tick(1, S4), + ok. + + +delivery_query_returns_deliveries_test(_) -> + Tag = atom_to_binary(?FUNCTION_NAME, utf8), + Cid = {Tag, self()}, + Commands = [ + rabbit_fifo_v0:make_checkout(Cid, {auto, 5, simple_prefetch}, #{}), + rabbit_fifo_v0:make_enqueue(self(), 1, one), + rabbit_fifo_v0:make_enqueue(self(), 2, two), + rabbit_fifo_v0:make_enqueue(self(), 3, tre), + rabbit_fifo_v0:make_enqueue(self(), 4, for) + ], + Indexes = lists:seq(1, length(Commands)), + Entries = lists:zip(Indexes, Commands), + {State, _Effects} = run_log(test_init(help), Entries), + % 3 deliveries are returned + [{0, {_, one}}] = rabbit_fifo_v0:get_checked_out(Cid, 0, 0, State), + [_, _, _] = rabbit_fifo_v0:get_checked_out(Cid, 1, 3, State), + ok. + +pending_enqueue_is_enqueued_on_down_test(_) -> + Cid = {<<"cid">>, self()}, + Pid = self(), + {State0, _} = enq(1, 2, first, test_init(test)), + {State1, _, _} = apply(meta(2), {down, Pid, noproc}, State0), + {_State2, {dequeue, {0, {_, first}}, 0}, _} = + apply(meta(3), rabbit_fifo_v0:make_checkout(Cid, {dequeue, settled}, #{}), State1), + ok. + +duplicate_delivery_test(_) -> + {State0, _} = enq(1, 1, first, test_init(test)), + {#?STATE{ra_indexes = RaIdxs, + messages = Messages}, _} = enq(2, 1, first, State0), + ?assertEqual(1, rabbit_fifo_index:size(RaIdxs)), + ?assertEqual(1, maps:size(Messages)), + ok. + +state_enter_file_handle_leader_reservation_test(_) -> + S0 = init(#{name => the_name, + queue_resource => rabbit_misc:r(<<"/">>, queue, <<"test">>), + become_leader_handler => {m, f, [a]}}), + + Resource = {resource, <<"/">>, queue, <<"test">>}, + Effects = rabbit_fifo_v0:state_enter(leader, S0), + ?assertEqual([ + {mod_call, m, f, [a, the_name]}, + {mod_call, rabbit_quorum_queue, file_handle_leader_reservation, [Resource]} + ], Effects), + ok. + +state_enter_file_handle_other_reservation_test(_) -> + S0 = init(#{name => the_name, + queue_resource => rabbit_misc:r(<<"/">>, queue, <<"test">>)}), + Effects = rabbit_fifo_v0:state_enter(other, S0), + ?assertEqual([ + {mod_call, rabbit_quorum_queue, file_handle_other_reservation, []} + ], + Effects), + ok. + +state_enter_monitors_and_notifications_test(_) -> + Oth = spawn(fun () -> ok end), + {State0, _} = enq(1, 1, first, test_init(test)), + Cid = {<<"adf">>, self()}, + OthCid = {<<"oth">>, Oth}, + {State1, _} = check(Cid, 2, State0), + {State, _} = check(OthCid, 3, State1), + Self = self(), + Effects = rabbit_fifo_v0:state_enter(leader, State), + + %% monitor all enqueuers and consumers + [{monitor, process, Self}, + {monitor, process, Oth}] = + lists:filter(fun ({monitor, process, _}) -> true; + (_) -> false + end, Effects), + [{send_msg, Self, leader_change, ra_event}, + {send_msg, Oth, leader_change, ra_event}] = + lists:filter(fun ({send_msg, _, leader_change, ra_event}) -> true; + (_) -> false + end, Effects), + ?ASSERT_EFF({monitor, process, _}, Effects), + ok. + +purge_test(_) -> + Cid = {<<"purge_test">>, self()}, + {State1, _} = enq(1, 1, first, test_init(test)), + {State2, {purge, 1}, _} = apply(meta(2), rabbit_fifo_v0:make_purge(), State1), + {State3, _} = enq(3, 2, second, State2), + % get returns a reply value + {_State4, {dequeue, {0, {_, second}}, _}, [{monitor, _, _}]} = + apply(meta(4), rabbit_fifo_v0:make_checkout(Cid, {dequeue, unsettled}, #{}), State3), + ok. + +purge_with_checkout_test(_) -> + Cid = {<<"purge_test">>, self()}, + {State0, _} = check_auto(Cid, 1, test_init(?FUNCTION_NAME)), + {State1, _} = enq(2, 1, <<"first">>, State0), + {State2, _} = enq(3, 2, <<"second">>, State1), + %% assert message bytes are non zero + ?assert(State2#?STATE.msg_bytes_checkout > 0), + ?assert(State2#?STATE.msg_bytes_enqueue > 0), + {State3, {purge, 1}, _} = apply(meta(2), rabbit_fifo_v0:make_purge(), State2), + ?assert(State2#?STATE.msg_bytes_checkout > 0), + ?assertEqual(0, State3#?STATE.msg_bytes_enqueue), + ?assertEqual(1, rabbit_fifo_index:size(State3#?STATE.ra_indexes)), + #consumer{checked_out = Checked} = maps:get(Cid, State3#?STATE.consumers), + ?assertEqual(1, maps:size(Checked)), + ok. + +down_noproc_returns_checked_out_in_order_test(_) -> + S0 = test_init(?FUNCTION_NAME), + %% enqueue 100 + S1 = lists:foldl(fun (Num, FS0) -> + {FS, _} = enq(Num, Num, Num, FS0), + FS + end, S0, lists:seq(1, 100)), + ?assertEqual(100, maps:size(S1#?STATE.messages)), + Cid = {<<"cid">>, self()}, + {S2, _} = check(Cid, 101, 1000, S1), + #consumer{checked_out = Checked} = maps:get(Cid, S2#?STATE.consumers), + ?assertEqual(100, maps:size(Checked)), + %% simulate down + {S, _, _} = apply(meta(102), {down, self(), noproc}, S2), + Returns = lqueue:to_list(S#?STATE.returns), + ?assertEqual(100, length(Returns)), + ?assertEqual(0, maps:size(S#?STATE.consumers)), + %% validate returns are in order + ?assertEqual(lists:sort(Returns), Returns), + ok. + +down_noconnection_returns_checked_out_test(_) -> + S0 = test_init(?FUNCTION_NAME), + NumMsgs = 20, + S1 = lists:foldl(fun (Num, FS0) -> + {FS, _} = enq(Num, Num, Num, FS0), + FS + end, S0, lists:seq(1, NumMsgs)), + ?assertEqual(NumMsgs, maps:size(S1#?STATE.messages)), + Cid = {<<"cid">>, self()}, + {S2, _} = check(Cid, 101, 1000, S1), + #consumer{checked_out = Checked} = maps:get(Cid, S2#?STATE.consumers), + ?assertEqual(NumMsgs, maps:size(Checked)), + %% simulate down + {S, _, _} = apply(meta(102), {down, self(), noconnection}, S2), + Returns = lqueue:to_list(S#?STATE.returns), + ?assertEqual(NumMsgs, length(Returns)), + ?assertMatch(#consumer{checked_out = Ch} + when map_size(Ch) == 0, + maps:get(Cid, S#?STATE.consumers)), + %% validate returns are in order + ?assertEqual(lists:sort(Returns), Returns), + ok. + +single_active_consumer_basic_get_test(_) -> + Cid = {?FUNCTION_NAME, self()}, + State0 = init(#{name => ?FUNCTION_NAME, + queue_resource => rabbit_misc:r("/", queue, + atom_to_binary(?FUNCTION_NAME, utf8)), + release_cursor_interval => 0, + single_active_consumer_on => true}), + ?assertEqual(single_active, State0#?STATE.cfg#cfg.consumer_strategy), + ?assertEqual(0, map_size(State0#?STATE.consumers)), + {State1, _} = enq(1, 1, first, State0), + {_State, {error, unsupported}} = + apply(meta(2), rabbit_fifo_v0:make_checkout(Cid, {dequeue, unsettled}, #{}), + State1), + ok. + +single_active_consumer_test(_) -> + State0 = init(#{name => ?FUNCTION_NAME, + queue_resource => rabbit_misc:r("/", queue, + atom_to_binary(?FUNCTION_NAME, utf8)), + release_cursor_interval => 0, + single_active_consumer_on => true}), + ?assertEqual(single_active, State0#?STATE.cfg#cfg.consumer_strategy), + ?assertEqual(0, map_size(State0#?STATE.consumers)), + + % adding some consumers + AddConsumer = fun(CTag, State) -> + {NewState, _, _} = apply( + meta(1), + make_checkout({CTag, self()}, + {once, 1, simple_prefetch}, + #{}), + State), + NewState + end, + State1 = lists:foldl(AddConsumer, State0, + [<<"ctag1">>, <<"ctag2">>, <<"ctag3">>, <<"ctag4">>]), + C1 = {<<"ctag1">>, self()}, + C2 = {<<"ctag2">>, self()}, + C3 = {<<"ctag3">>, self()}, + C4 = {<<"ctag4">>, self()}, + + % the first registered consumer is the active one, the others are waiting + ?assertEqual(1, map_size(State1#?STATE.consumers)), + ?assertMatch(#{C1 := _}, State1#?STATE.consumers), + ?assertEqual(3, length(State1#?STATE.waiting_consumers)), + ?assertNotEqual(false, lists:keyfind(C2, 1, State1#?STATE.waiting_consumers)), + ?assertNotEqual(false, lists:keyfind(C3, 1, State1#?STATE.waiting_consumers)), + ?assertNotEqual(false, lists:keyfind(C4, 1, State1#?STATE.waiting_consumers)), + + % cancelling a waiting consumer + {State2, _, Effects1} = apply(meta(2), + make_checkout(C3, cancel, #{}), + State1), + % the active consumer should still be in place + ?assertEqual(1, map_size(State2#?STATE.consumers)), + ?assertMatch(#{C1 := _}, State2#?STATE.consumers), + % the cancelled consumer has been removed from waiting consumers + ?assertEqual(2, length(State2#?STATE.waiting_consumers)), + ?assertNotEqual(false, lists:keyfind(C2, 1, State2#?STATE.waiting_consumers)), + ?assertNotEqual(false, lists:keyfind(C4, 1, State2#?STATE.waiting_consumers)), + % there are some effects to unregister the consumer + ?ASSERT_EFF({mod_call, rabbit_quorum_queue, + cancel_consumer_handler, [_, C]}, C == C3, Effects1), + + % cancelling the active consumer + {State3, _, Effects2} = apply(meta(3), + make_checkout(C1, cancel, #{}), + State2), + % the second registered consumer is now the active one + ?assertEqual(1, map_size(State3#?STATE.consumers)), + ?assertMatch(#{C2 := _}, State3#?STATE.consumers), + % the new active consumer is no longer in the waiting list + ?assertEqual(1, length(State3#?STATE.waiting_consumers)), + ?assertNotEqual(false, lists:keyfind(C4, 1, + State3#?STATE.waiting_consumers)), + %% should have a cancel consumer handler mod_call effect and + %% an active new consumer effect + ?ASSERT_EFF({mod_call, rabbit_quorum_queue, + cancel_consumer_handler, [_, C]}, C == C1, Effects2), + ?ASSERT_EFF({mod_call, rabbit_quorum_queue, + update_consumer_handler, _}, Effects2), + + % cancelling the active consumer + {State4, _, Effects3} = apply(meta(4), + make_checkout(C2, cancel, #{}), + State3), + % the last waiting consumer became the active one + ?assertEqual(1, map_size(State4#?STATE.consumers)), + ?assertMatch(#{C4 := _}, State4#?STATE.consumers), + % the waiting consumer list is now empty + ?assertEqual(0, length(State4#?STATE.waiting_consumers)), + % there are some effects to unregister the consumer and + % to update the new active one (metrics) + ?ASSERT_EFF({mod_call, rabbit_quorum_queue, + cancel_consumer_handler, [_, C]}, C == C2, Effects3), + ?ASSERT_EFF({mod_call, rabbit_quorum_queue, + update_consumer_handler, _}, Effects3), + + % cancelling the last consumer + {State5, _, Effects4} = apply(meta(5), + make_checkout(C4, cancel, #{}), + State4), + % no active consumer anymore + ?assertEqual(0, map_size(State5#?STATE.consumers)), + % still nothing in the waiting list + ?assertEqual(0, length(State5#?STATE.waiting_consumers)), + % there is an effect to unregister the consumer + queue inactive effect + ?ASSERT_EFF({mod_call, rabbit_quorum_queue, + cancel_consumer_handler, _}, Effects4), + + ok. + +single_active_consumer_cancel_consumer_when_channel_is_down_test(_) -> + State0 = init(#{name => ?FUNCTION_NAME, + queue_resource => rabbit_misc:r("/", queue, + atom_to_binary(?FUNCTION_NAME, utf8)), + release_cursor_interval => 0, + single_active_consumer_on => true}), + + DummyFunction = fun() -> ok end, + Pid1 = spawn(DummyFunction), + Pid2 = spawn(DummyFunction), + Pid3 = spawn(DummyFunction), + + [C1, C2, C3, C4] = Consumers = + [{<<"ctag1">>, Pid1}, {<<"ctag2">>, Pid2}, + {<<"ctag3">>, Pid2}, {<<"ctag4">>, Pid3}], + % adding some consumers + AddConsumer = fun({CTag, ChannelId}, State) -> + {NewState, _, _} = apply( + #{index => 1}, + make_checkout({CTag, ChannelId}, {once, 1, simple_prefetch}, #{}), + State), + NewState + end, + State1 = lists:foldl(AddConsumer, State0, Consumers), + + % the channel of the active consumer goes down + {State2, _, Effects} = apply(#{index => 2}, {down, Pid1, noproc}, State1), + % fell back to another consumer + ?assertEqual(1, map_size(State2#?STATE.consumers)), + % there are still waiting consumers + ?assertEqual(2, length(State2#?STATE.waiting_consumers)), + % effects to unregister the consumer and + % to update the new active one (metrics) are there + ?ASSERT_EFF({mod_call, rabbit_quorum_queue, + cancel_consumer_handler, [_, C]}, C == C1, Effects), + ?ASSERT_EFF({mod_call, rabbit_quorum_queue, + update_consumer_handler, _}, Effects), + + % the channel of the active consumer and a waiting consumer goes down + {State3, _, Effects2} = apply(#{index => 3}, {down, Pid2, noproc}, State2), + % fell back to another consumer + ?assertEqual(1, map_size(State3#?STATE.consumers)), + % no more waiting consumer + ?assertEqual(0, length(State3#?STATE.waiting_consumers)), + % effects to cancel both consumers of this channel + effect to update the new active one (metrics) + ?ASSERT_EFF({mod_call, rabbit_quorum_queue, + cancel_consumer_handler, [_, C]}, C == C2, Effects2), + ?ASSERT_EFF({mod_call, rabbit_quorum_queue, + cancel_consumer_handler, [_, C]}, C == C3, Effects2), + ?ASSERT_EFF({mod_call, rabbit_quorum_queue, + update_consumer_handler, _}, Effects2), + + % the last channel goes down + {State4, _, Effects3} = apply(#{index => 4}, {down, Pid3, doesnotmatter}, State3), + % no more consumers + ?assertEqual(0, map_size(State4#?STATE.consumers)), + ?assertEqual(0, length(State4#?STATE.waiting_consumers)), + % there is an effect to unregister the consumer + queue inactive effect + ?ASSERT_EFF({mod_call, rabbit_quorum_queue, + cancel_consumer_handler, [_, C]}, C == C4, Effects3), + + ok. + +single_active_returns_messages_on_noconnection_test(_) -> + R = rabbit_misc:r("/", queue, atom_to_binary(?FUNCTION_NAME, utf8)), + State0 = init(#{name => ?FUNCTION_NAME, + queue_resource => R, + release_cursor_interval => 0, + single_active_consumer_on => true}), + Meta = #{index => 1}, + Nodes = [n1], + ConsumerIds = [{_, DownPid}] = + [begin + B = atom_to_binary(N, utf8), + {<<"ctag_", B/binary>>, + test_util:fake_pid(N)} + end || N <- Nodes], + % adding some consumers + State1 = lists:foldl( + fun(CId, Acc0) -> + {Acc, _, _} = + apply(Meta, + make_checkout(CId, + {once, 1, simple_prefetch}, #{}), + Acc0), + Acc + end, State0, ConsumerIds), + {State2, _} = enq(4, 1, msg1, State1), + % simulate node goes down + {State3, _, _} = apply(meta(5), {down, DownPid, noconnection}, State2), + %% assert the consumer is up + ?assertMatch([_], lqueue:to_list(State3#?STATE.returns)), + ?assertMatch([{_, #consumer{checked_out = Checked}}] + when map_size(Checked) == 0, + State3#?STATE.waiting_consumers), + + ok. + +single_active_consumer_replaces_consumer_when_down_noconnection_test(_) -> + R = rabbit_misc:r("/", queue, atom_to_binary(?FUNCTION_NAME, utf8)), + State0 = init(#{name => ?FUNCTION_NAME, + queue_resource => R, + release_cursor_interval => 0, + single_active_consumer_on => true}), + Meta = #{index => 1}, + Nodes = [n1, n2, node()], + ConsumerIds = [C1 = {_, DownPid}, C2, _C3] = + [begin + B = atom_to_binary(N, utf8), + {<<"ctag_", B/binary>>, + test_util:fake_pid(N)} + end || N <- Nodes], + % adding some consumers + State1a = lists:foldl( + fun(CId, Acc0) -> + {Acc, _, _} = + apply(Meta, + make_checkout(CId, + {once, 1, simple_prefetch}, #{}), + Acc0), + Acc + end, State0, ConsumerIds), + + %% assert the consumer is up + ?assertMatch(#{C1 := #consumer{status = up}}, + State1a#?STATE.consumers), + + {State1, _} = enq(10, 1, msg, State1a), + + % simulate node goes down + {State2, _, _} = apply(meta(5), {down, DownPid, noconnection}, State1), + + %% assert a new consumer is in place and it is up + ?assertMatch([{C2, #consumer{status = up, + checked_out = Ch}}] + when map_size(Ch) == 1, + maps:to_list(State2#?STATE.consumers)), + + %% the disconnected consumer has been returned to waiting + ?assert(lists:any(fun ({C,_}) -> C =:= C1 end, + State2#?STATE.waiting_consumers)), + ?assertEqual(2, length(State2#?STATE.waiting_consumers)), + + % simulate node comes back up + {State3, _, _} = apply(#{index => 2}, {nodeup, node(DownPid)}, State2), + + %% the consumer is still active and the same as before + ?assertMatch([{C2, #consumer{status = up}}], + maps:to_list(State3#?STATE.consumers)), + % the waiting consumers should be un-suspected + ?assertEqual(2, length(State3#?STATE.waiting_consumers)), + lists:foreach(fun({_, #consumer{status = Status}}) -> + ?assert(Status /= suspected_down) + end, State3#?STATE.waiting_consumers), + ok. + +single_active_consumer_all_disconnected_test(_) -> + R = rabbit_misc:r("/", queue, atom_to_binary(?FUNCTION_NAME, utf8)), + State0 = init(#{name => ?FUNCTION_NAME, + queue_resource => R, + release_cursor_interval => 0, + single_active_consumer_on => true}), + Meta = #{index => 1}, + Nodes = [n1, n2], + ConsumerIds = [C1 = {_, C1Pid}, C2 = {_, C2Pid}] = + [begin + B = atom_to_binary(N, utf8), + {<<"ctag_", B/binary>>, + test_util:fake_pid(N)} + end || N <- Nodes], + % adding some consumers + State1 = lists:foldl( + fun(CId, Acc0) -> + {Acc, _, _} = + apply(Meta, + make_checkout(CId, + {once, 1, simple_prefetch}, #{}), + Acc0), + Acc + end, State0, ConsumerIds), + + %% assert the consumer is up + ?assertMatch(#{C1 := #consumer{status = up}}, State1#?STATE.consumers), + + % simulate node goes down + {State2, _, _} = apply(meta(5), {down, C1Pid, noconnection}, State1), + %% assert the consumer fails over to the consumer on n2 + ?assertMatch(#{C2 := #consumer{status = up}}, State2#?STATE.consumers), + {State3, _, _} = apply(meta(6), {down, C2Pid, noconnection}, State2), + %% assert these no active consumer after both nodes are maked as down + ?assertMatch([], maps:to_list(State3#?STATE.consumers)), + %% n2 comes back + {State4, _, _} = apply(meta(7), {nodeup, node(C2Pid)}, State3), + %% ensure n2 is the active consumer as this node as been registered + %% as up again + ?assertMatch([{{<<"ctag_n2">>, _}, #consumer{status = up, + credit = 1}}], + maps:to_list(State4#?STATE.consumers)), + ok. + +single_active_consumer_state_enter_leader_include_waiting_consumers_test(_) -> + State0 = init(#{name => ?FUNCTION_NAME, + queue_resource => + rabbit_misc:r("/", queue, + atom_to_binary(?FUNCTION_NAME, utf8)), + release_cursor_interval => 0, + single_active_consumer_on => true}), + + DummyFunction = fun() -> ok end, + Pid1 = spawn(DummyFunction), + Pid2 = spawn(DummyFunction), + Pid3 = spawn(DummyFunction), + + Meta = #{index => 1}, + % adding some consumers + AddConsumer = fun({CTag, ChannelId}, State) -> + {NewState, _, _} = apply( + Meta, + make_checkout({CTag, ChannelId}, + {once, 1, simple_prefetch}, #{}), + State), + NewState + end, + State1 = lists:foldl(AddConsumer, State0, + [{<<"ctag1">>, Pid1}, {<<"ctag2">>, Pid2}, {<<"ctag3">>, Pid2}, {<<"ctag4">>, Pid3}]), + + Effects = rabbit_fifo_v0:state_enter(leader, State1), + %% 2 effects for each consumer process (channel process), 1 effect for the node, + %% 1 effect for file handle reservation + ?assertEqual(2 * 3 + 1 + 1, length(Effects)). + +single_active_consumer_state_enter_eol_include_waiting_consumers_test(_) -> + Resource = rabbit_misc:r("/", queue, atom_to_binary(?FUNCTION_NAME, utf8)), + State0 = init(#{name => ?FUNCTION_NAME, + queue_resource => Resource, + release_cursor_interval => 0, + single_active_consumer_on => true}), + + DummyFunction = fun() -> ok end, + Pid1 = spawn(DummyFunction), + Pid2 = spawn(DummyFunction), + Pid3 = spawn(DummyFunction), + + Meta = #{index => 1}, + % adding some consumers + AddConsumer = fun({CTag, ChannelId}, State) -> + {NewState, _, _} = apply( + Meta, + make_checkout({CTag, ChannelId}, + {once, 1, simple_prefetch}, #{}), + State), + NewState + end, + State1 = lists:foldl(AddConsumer, State0, + [{<<"ctag1">>, Pid1}, {<<"ctag2">>, Pid2}, + {<<"ctag3">>, Pid2}, {<<"ctag4">>, Pid3}]), + + Effects = rabbit_fifo_v0:state_enter(eol, State1), + %% 1 effect for each consumer process (channel process), + %% 1 effect for file handle reservation + ?assertEqual(4, length(Effects)). + +query_consumers_test(_) -> + State0 = init(#{name => ?FUNCTION_NAME, + queue_resource => rabbit_misc:r("/", queue, + atom_to_binary(?FUNCTION_NAME, utf8)), + release_cursor_interval => 0, + single_active_consumer_on => false}), + + % adding some consumers + AddConsumer = fun(CTag, State) -> + {NewState, _, _} = apply( + #{index => 1}, + make_checkout({CTag, self()}, + {once, 1, simple_prefetch}, #{}), + State), + NewState + end, + State1 = lists:foldl(AddConsumer, State0, [<<"ctag1">>, <<"ctag2">>, <<"ctag3">>, <<"ctag4">>]), + Consumers0 = State1#?STATE.consumers, + Consumer = maps:get({<<"ctag2">>, self()}, Consumers0), + Consumers1 = maps:put({<<"ctag2">>, self()}, + Consumer#consumer{status = suspected_down}, Consumers0), + State2 = State1#?STATE{consumers = Consumers1}, + + ?assertEqual(4, rabbit_fifo_v0:query_consumer_count(State2)), + Consumers2 = rabbit_fifo_v0:query_consumers(State2), + ?assertEqual(4, maps:size(Consumers2)), + maps:fold(fun(_Key, {Pid, Tag, _, _, Active, ActivityStatus, _, _}, _Acc) -> + ?assertEqual(self(), Pid), + case Tag of + <<"ctag2">> -> + ?assertNot(Active), + ?assertEqual(suspected_down, ActivityStatus); + _ -> + ?assert(Active), + ?assertEqual(up, ActivityStatus) + end + end, [], Consumers2). + +query_consumers_when_single_active_consumer_is_on_test(_) -> + State0 = init(#{name => ?FUNCTION_NAME, + queue_resource => rabbit_misc:r("/", queue, + atom_to_binary(?FUNCTION_NAME, utf8)), + release_cursor_interval => 0, + single_active_consumer_on => true}), + Meta = #{index => 1}, + % adding some consumers + AddConsumer = fun(CTag, State) -> + {NewState, _, _} = apply( + Meta, + make_checkout({CTag, self()}, + {once, 1, simple_prefetch}, #{}), + State), + NewState + end, + State1 = lists:foldl(AddConsumer, State0, [<<"ctag1">>, <<"ctag2">>, <<"ctag3">>, <<"ctag4">>]), + + ?assertEqual(4, rabbit_fifo_v0:query_consumer_count(State1)), + Consumers = rabbit_fifo_v0:query_consumers(State1), + ?assertEqual(4, maps:size(Consumers)), + maps:fold(fun(_Key, {Pid, Tag, _, _, Active, ActivityStatus, _, _}, _Acc) -> + ?assertEqual(self(), Pid), + case Tag of + <<"ctag1">> -> + ?assert(Active), + ?assertEqual(single_active, ActivityStatus); + _ -> + ?assertNot(Active), + ?assertEqual(waiting, ActivityStatus) + end + end, [], Consumers). + +active_flag_updated_when_consumer_suspected_unsuspected_test(_) -> + State0 = init(#{name => ?FUNCTION_NAME, + queue_resource => rabbit_misc:r("/", queue, + atom_to_binary(?FUNCTION_NAME, utf8)), + release_cursor_interval => 0, + single_active_consumer_on => false}), + + DummyFunction = fun() -> ok end, + Pid1 = spawn(DummyFunction), + Pid2 = spawn(DummyFunction), + Pid3 = spawn(DummyFunction), + + % adding some consumers + AddConsumer = fun({CTag, ChannelId}, State) -> + {NewState, _, _} = + apply( + #{index => 1}, + rabbit_fifo_v0:make_checkout({CTag, ChannelId}, + {once, 1, simple_prefetch}, + #{}), + State), + NewState + end, + State1 = lists:foldl(AddConsumer, State0, + [{<<"ctag1">>, Pid1}, {<<"ctag2">>, Pid2}, {<<"ctag3">>, Pid2}, {<<"ctag4">>, Pid3}]), + + {State2, _, Effects2} = apply(#{index => 3}, {down, Pid1, noconnection}, State1), + % 1 effect to update the metrics of each consumer (they belong to the same node), 1 more effect to monitor the node + ?assertEqual(4 + 1, length(Effects2)), + + {_, _, Effects3} = apply(#{index => 4}, {nodeup, node(self())}, State2), + % for each consumer: 1 effect to update the metrics, 1 effect to monitor the consumer PID + ?assertEqual(4 + 4, length(Effects3)). + +active_flag_not_updated_when_consumer_suspected_unsuspected_and_single_active_consumer_is_on_test(_) -> + State0 = init(#{name => ?FUNCTION_NAME, + queue_resource => rabbit_misc:r("/", queue, + atom_to_binary(?FUNCTION_NAME, utf8)), + release_cursor_interval => 0, + single_active_consumer_on => true}), + + DummyFunction = fun() -> ok end, + Pid1 = spawn(DummyFunction), + Pid2 = spawn(DummyFunction), + Pid3 = spawn(DummyFunction), + + % adding some consumers + AddConsumer = fun({CTag, ChannelId}, State) -> + {NewState, _, _} = apply( + #{index => 1}, + make_checkout({CTag, ChannelId}, + {once, 1, simple_prefetch}, #{}), + State), + NewState + end, + State1 = lists:foldl(AddConsumer, State0, + [{<<"ctag1">>, Pid1}, {<<"ctag2">>, Pid2}, + {<<"ctag3">>, Pid2}, {<<"ctag4">>, Pid3}]), + + {State2, _, Effects2} = apply(#{index => 2}, {down, Pid1, noconnection}, State1), + % one monitor and one consumer status update (deactivated) + ?assertEqual(3, length(Effects2)), + + {_, _, Effects3} = apply(#{index => 3}, {nodeup, node(self())}, State2), + % for each consumer: 1 effect to monitor the consumer PID + ?assertEqual(5, length(Effects3)). + +single_active_cancelled_with_unacked_test(_) -> + State0 = init(#{name => ?FUNCTION_NAME, + queue_resource => rabbit_misc:r("/", queue, + atom_to_binary(?FUNCTION_NAME, utf8)), + release_cursor_interval => 0, + single_active_consumer_on => true}), + + C1 = {<<"ctag1">>, self()}, + C2 = {<<"ctag2">>, self()}, + % adding some consumers + AddConsumer = fun(C, S0) -> + {S, _, _} = apply( + meta(1), + make_checkout(C, + {auto, 1, simple_prefetch}, + #{}), + S0), + S + end, + State1 = lists:foldl(AddConsumer, State0, [C1, C2]), + + %% enqueue 2 messages + {State2, _Effects2} = enq(3, 1, msg1, State1), + {State3, _Effects3} = enq(4, 2, msg2, State2), + %% one should be checked ou to C1 + %% cancel C1 + {State4, _, _} = apply(meta(5), + make_checkout(C1, cancel, #{}), + State3), + %% C2 should be the active consumer + ?assertMatch(#{C2 := #consumer{status = up, + checked_out = #{0 := _}}}, + State4#?STATE.consumers), + %% C1 should be a cancelled consumer + ?assertMatch(#{C1 := #consumer{status = cancelled, + lifetime = once, + checked_out = #{0 := _}}}, + State4#?STATE.consumers), + ?assertMatch([], State4#?STATE.waiting_consumers), + + %% Ack both messages + {State5, _Effects5} = settle(C1, 1, 0, State4), + %% C1 should now be cancelled + {State6, _Effects6} = settle(C2, 2, 0, State5), + + %% C2 should remain + ?assertMatch(#{C2 := #consumer{status = up}}, + State6#?STATE.consumers), + %% C1 should be gone + ?assertNotMatch(#{C1 := _}, + State6#?STATE.consumers), + ?assertMatch([], State6#?STATE.waiting_consumers), + ok. + +single_active_with_credited_test(_) -> + State0 = init(#{name => ?FUNCTION_NAME, + queue_resource => rabbit_misc:r("/", queue, + atom_to_binary(?FUNCTION_NAME, utf8)), + release_cursor_interval => 0, + single_active_consumer_on => true}), + + C1 = {<<"ctag1">>, self()}, + C2 = {<<"ctag2">>, self()}, + % adding some consumers + AddConsumer = fun(C, S0) -> + {S, _, _} = apply( + meta(1), + make_checkout(C, + {auto, 0, credited}, + #{}), + S0), + S + end, + State1 = lists:foldl(AddConsumer, State0, [C1, C2]), + + %% add some credit + C1Cred = rabbit_fifo_v0:make_credit(C1, 5, 0, false), + {State2, _, _Effects2} = apply(meta(3), C1Cred, State1), + C2Cred = rabbit_fifo_v0:make_credit(C2, 4, 0, false), + {State3, _} = apply(meta(4), C2Cred, State2), + %% both consumers should have credit + ?assertMatch(#{C1 := #consumer{credit = 5}}, + State3#?STATE.consumers), + ?assertMatch([{C2, #consumer{credit = 4}}], + State3#?STATE.waiting_consumers), + ok. + +purge_nodes_test(_) -> + Node = purged@node, + ThisNode = node(), + EnqPid = test_util:fake_pid(Node), + EnqPid2 = test_util:fake_pid(node()), + ConPid = test_util:fake_pid(Node), + Cid = {<<"tag">>, ConPid}, + % WaitingPid = test_util:fake_pid(Node), + + State0 = init(#{name => ?FUNCTION_NAME, + queue_resource => rabbit_misc:r("/", queue, + atom_to_binary(?FUNCTION_NAME, utf8)), + single_active_consumer_on => false}), + {State1, _, _} = apply(meta(1), + rabbit_fifo_v0:make_enqueue(EnqPid, 1, msg1), + State0), + {State2, _, _} = apply(meta(2), + rabbit_fifo_v0:make_enqueue(EnqPid2, 1, msg2), + State1), + {State3, _} = check(Cid, 3, 1000, State2), + {State4, _, _} = apply(meta(4), + {down, EnqPid, noconnection}, + State3), + ?assertMatch( + [{mod_call, rabbit_quorum_queue, handle_tick, + [#resource{}, _Metrics, + [ThisNode, Node] + ]}] , rabbit_fifo_v0:tick(1, State4)), + %% assert there are both enqueuers and consumers + {State, _, _} = apply(meta(5), + rabbit_fifo_v0:make_purge_nodes([Node]), + State4), + + %% assert there are no enqueuers nor consumers + ?assertMatch(#?STATE{enqueuers = Enqs} when map_size(Enqs) == 1, State), + ?assertMatch(#?STATE{consumers = Cons} when map_size(Cons) == 0, State), + ?assertMatch( + [{mod_call, rabbit_quorum_queue, handle_tick, + [#resource{}, _Metrics, + [ThisNode] + ]}] , rabbit_fifo_v0:tick(1, State)), + ok. + +meta(Idx) -> + #{index => Idx, term => 1, + from => {make_ref(), self()}}. + +enq(Idx, MsgSeq, Msg, State) -> + strip_reply( + apply(meta(Idx), rabbit_fifo_v0:make_enqueue(self(), MsgSeq, Msg), State)). + +deq(Idx, Cid, Settlement, State0) -> + {State, {dequeue, {MsgId, Msg}, _}, _} = + apply(meta(Idx), + rabbit_fifo_v0:make_checkout(Cid, {dequeue, Settlement}, #{}), + State0), + {State, {MsgId, Msg}}. + +check_n(Cid, Idx, N, State) -> + strip_reply( + apply(meta(Idx), + rabbit_fifo_v0:make_checkout(Cid, {auto, N, simple_prefetch}, #{}), + State)). + +check(Cid, Idx, State) -> + strip_reply( + apply(meta(Idx), + rabbit_fifo_v0:make_checkout(Cid, {once, 1, simple_prefetch}, #{}), + State)). + +check_auto(Cid, Idx, State) -> + strip_reply( + apply(meta(Idx), + rabbit_fifo_v0:make_checkout(Cid, {auto, 1, simple_prefetch}, #{}), + State)). + +check(Cid, Idx, Num, State) -> + strip_reply( + apply(meta(Idx), + rabbit_fifo_v0:make_checkout(Cid, {auto, Num, simple_prefetch}, #{}), + State)). + +settle(Cid, Idx, MsgId, State) -> + strip_reply(apply(meta(Idx), rabbit_fifo_v0:make_settle(Cid, [MsgId]), State)). + +credit(Cid, Idx, Credit, DelCnt, Drain, State) -> + strip_reply(apply(meta(Idx), rabbit_fifo_v0:make_credit(Cid, Credit, DelCnt, Drain), + State)). + +strip_reply({State, _, Effects}) -> + {State, Effects}. + +run_log(InitState, Entries) -> + lists:foldl(fun ({Idx, E}, {Acc0, Efx0}) -> + case apply(meta(Idx), E, Acc0) of + {Acc, _, Efx} when is_list(Efx) -> + {Acc, Efx0 ++ Efx}; + {Acc, _, Efx} -> + {Acc, Efx0 ++ [Efx]}; + {Acc, _} -> + {Acc, Efx0} + end + end, {InitState, []}, Entries). + + +%% AUX Tests + +aux_test(_) -> + _ = ra_machine_ets:start_link(), + Aux0 = init_aux(aux_test), + MacState = init(#{name => aux_test, + queue_resource => + rabbit_misc:r(<<"/">>, queue, <<"test">>)}), + ok = meck:new(ra_log, []), + Log = mock_log, + meck:expect(ra_log, last_index_term, fun (_) -> {0, 0} end), + {no_reply, Aux, mock_log} = handle_aux(leader, cast, active, Aux0, + Log, MacState), + {no_reply, _Aux, mock_log} = handle_aux(leader, cast, tick, Aux, + Log, MacState), + [X] = ets:lookup(rabbit_fifo_usage, aux_test), + meck:unload(), + ?assert(X > 0.0), + ok. + +%% Utility + +init(Conf) -> rabbit_fifo_v0:init(Conf). +apply(Meta, Entry, State) -> rabbit_fifo_v0:apply(Meta, Entry, State). +init_aux(Conf) -> rabbit_fifo_v0:init_aux(Conf). +handle_aux(S, T, C, A, L, M) -> rabbit_fifo_v0:handle_aux(S, T, C, A, L, M). +make_checkout(C, S, M) -> rabbit_fifo_v0:make_checkout(C, S, M). diff --git a/deps/rabbit/test/rabbit_foo_protocol_connection_info.erl b/deps/rabbit/test/rabbit_foo_protocol_connection_info.erl new file mode 100644 index 0000000000..937558aba8 --- /dev/null +++ b/deps/rabbit/test/rabbit_foo_protocol_connection_info.erl @@ -0,0 +1,25 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2019-2020 VMware, Inc. or its affiliates. All rights reserved. +%% +-module(rabbit_foo_protocol_connection_info). + +%% Dummy module to test authentication context propagation + +%% API +-export([additional_authn_params/4]). + +additional_authn_params(_Creds, _VHost, _Pid, Infos) -> + case proplists:get_value(variable_map, Infos, undefined) of + VariableMap when is_map(VariableMap) -> + case maps:get(<<"key1">>, VariableMap, []) of + Value when is_binary(Value)-> + [{key1, Value}]; + [] -> + [] + end; + _ -> + [] + end. diff --git a/deps/rabbit/test/rabbit_ha_test_consumer.erl b/deps/rabbit/test/rabbit_ha_test_consumer.erl new file mode 100644 index 0000000000..2467e40028 --- /dev/null +++ b/deps/rabbit/test/rabbit_ha_test_consumer.erl @@ -0,0 +1,108 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. +%% +-module(rabbit_ha_test_consumer). + +-include_lib("amqp_client/include/amqp_client.hrl"). + +-export([await_response/1, create/5, start/6]). + +await_response(ConsumerPid) -> + case receive {ConsumerPid, Response} -> Response end of + {error, Reason} -> erlang:error(Reason); + ok -> ok + end. + +create(Channel, Queue, TestPid, CancelOnFailover, ExpectingMsgs) -> + ConsumerPid = spawn_link(?MODULE, start, + [TestPid, Channel, Queue, CancelOnFailover, + ExpectingMsgs + 1, ExpectingMsgs]), + amqp_channel:subscribe( + Channel, consume_method(Queue, CancelOnFailover), ConsumerPid), + ConsumerPid. + +start(TestPid, Channel, Queue, CancelOnFailover, LowestSeen, MsgsToConsume) -> + error_logger:info_msg("consumer ~p on ~p awaiting ~w messages " + "(lowest seen = ~w, cancel-on-failover = ~w)~n", + [self(), Channel, MsgsToConsume, LowestSeen, + CancelOnFailover]), + run(TestPid, Channel, Queue, CancelOnFailover, LowestSeen, MsgsToConsume). + +run(TestPid, _Channel, _Queue, _CancelOnFailover, _LowestSeen, 0) -> + consumer_reply(TestPid, ok); +run(TestPid, Channel, Queue, CancelOnFailover, LowestSeen, MsgsToConsume) -> + receive + #'basic.consume_ok'{} -> + run(TestPid, Channel, Queue, + CancelOnFailover, LowestSeen, MsgsToConsume); + {Delivery = #'basic.deliver'{ redelivered = Redelivered }, + #amqp_msg{payload = Payload}} -> + MsgNum = list_to_integer(binary_to_list(Payload)), + + ack(Delivery, Channel), + + %% we can receive any message we've already seen and, + %% because of the possibility of multiple requeuings, we + %% might see these messages in any order. If we are seeing + %% a message again, we don't decrement the MsgsToConsume + %% counter. + if + MsgNum + 1 == LowestSeen -> + error_logger:info_msg("recording ~w left ~w", + [MsgNum, MsgsToConsume]), + run(TestPid, Channel, Queue, + CancelOnFailover, MsgNum, MsgsToConsume - 1); + MsgNum >= LowestSeen -> + error_logger:info_msg( + "consumer ~p on ~p ignoring redelivered msg ~p" + "lowest seen ~w~n", + [self(), Channel, MsgNum, LowestSeen]), + true = Redelivered, %% ASSERTION + run(TestPid, Channel, Queue, + CancelOnFailover, LowestSeen, MsgsToConsume); + true -> + %% We received a message we haven't seen before, + %% but it is not the next message in the expected + %% sequence. + consumer_reply(TestPid, + {error, {unexpected_message, MsgNum}}) + end; + #'basic.cancel'{} when CancelOnFailover -> + error_logger:info_msg("consumer ~p on ~p received basic.cancel: " + "resubscribing to ~p on ~p~n", + [self(), Channel, Queue, Channel]), + resubscribe(TestPid, Channel, Queue, CancelOnFailover, + LowestSeen, MsgsToConsume); + #'basic.cancel'{} -> + exit(cancel_received_without_cancel_on_failover) + end. + +%% +%% Private API +%% + +resubscribe(TestPid, Channel, Queue, CancelOnFailover, LowestSeen, + MsgsToConsume) -> + amqp_channel:subscribe( + Channel, consume_method(Queue, CancelOnFailover), self()), + ok = receive #'basic.consume_ok'{} -> ok + end, + error_logger:info_msg("re-subscripting consumer ~p on ~p complete " + "(received basic.consume_ok)", + [self(), Channel]), + start(TestPid, Channel, Queue, CancelOnFailover, LowestSeen, MsgsToConsume). + +consume_method(Queue, CancelOnFailover) -> + Args = [{<<"x-cancel-on-ha-failover">>, bool, CancelOnFailover}], + #'basic.consume'{queue = Queue, + arguments = Args}. + +ack(#'basic.deliver'{delivery_tag = DeliveryTag}, Channel) -> + amqp_channel:call(Channel, #'basic.ack'{delivery_tag = DeliveryTag}), + ok. + +consumer_reply(TestPid, Reply) -> + TestPid ! {self(), Reply}. diff --git a/deps/rabbit/test/rabbit_ha_test_producer.erl b/deps/rabbit/test/rabbit_ha_test_producer.erl new file mode 100644 index 0000000000..ed6969debe --- /dev/null +++ b/deps/rabbit/test/rabbit_ha_test_producer.erl @@ -0,0 +1,131 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. +%% +-module(rabbit_ha_test_producer). + +-export([await_response/1, start/6, create/5, create/6]). + +-include_lib("amqp_client/include/amqp_client.hrl"). + +await_response(ProducerPid) -> + error_logger:info_msg("waiting for producer pid ~p~n", [ProducerPid]), + case receive {ProducerPid, Response} -> Response end of + ok -> ok; + {error, _} = Else -> exit(Else); + Else -> exit({weird_response, Else}) + end. + +create(Channel, Queue, TestPid, Confirm, MsgsToSend) -> + create(Channel, Queue, TestPid, Confirm, MsgsToSend, acks). + +create(Channel, Queue, TestPid, Confirm, MsgsToSend, Mode) -> + AckNackMsgs = case Mode of + acks -> {ok, {error, received_nacks}}; + nacks -> {{error, received_acks}, ok} + end, + ProducerPid = spawn_link(?MODULE, start, [Channel, Queue, TestPid, + Confirm, MsgsToSend, AckNackMsgs]), + receive + {ProducerPid, started} -> ProducerPid + end. + +start(Channel, Queue, TestPid, Confirm, MsgsToSend, AckNackMsgs) -> + ConfirmState = + case Confirm of + true -> amqp_channel:register_confirm_handler(Channel, self()), + #'confirm.select_ok'{} = + amqp_channel:call(Channel, #'confirm.select'{}), + gb_trees:empty(); + false -> none + end, + TestPid ! {self(), started}, + error_logger:info_msg("publishing ~w msgs on ~p~n", [MsgsToSend, Channel]), + producer(Channel, Queue, TestPid, ConfirmState, MsgsToSend, AckNackMsgs). + +%% +%% Private API +%% + +producer(_Channel, _Queue, TestPid, none, 0, _AckNackMsgs) -> + TestPid ! {self(), ok}; +producer(Channel, _Queue, TestPid, ConfirmState, 0, {AckMsg, NackMsg}) -> + error_logger:info_msg("awaiting confirms on channel ~p~n", [Channel]), + Msg = case drain_confirms(none, ConfirmState) of + %% No acks or nacks + acks -> AckMsg; + nacks -> NackMsg; + mix -> {error, received_both_acks_and_nacks}; + {Nacks, CS} -> {error, {missing_confirms, Nacks, + lists:sort(gb_trees:keys(CS))}} + end, + TestPid ! {self(), Msg}; + +producer(Channel, Queue, TestPid, ConfirmState, MsgsToSend, AckNackMsgs) -> + Method = #'basic.publish'{exchange = <<"">>, + routing_key = Queue, + mandatory = false, + immediate = false}, + + ConfirmState1 = maybe_record_confirm(ConfirmState, Channel, MsgsToSend), + + amqp_channel:call(Channel, Method, + #amqp_msg{props = #'P_basic'{delivery_mode = 2}, + payload = list_to_binary( + integer_to_list(MsgsToSend))}), + + producer(Channel, Queue, TestPid, ConfirmState1, MsgsToSend - 1, AckNackMsgs). + +maybe_record_confirm(none, _, _) -> + none; +maybe_record_confirm(ConfirmState, Channel, MsgsToSend) -> + SeqNo = amqp_channel:next_publish_seqno(Channel), + gb_trees:insert(SeqNo, MsgsToSend, ConfirmState). + +drain_confirms(Collected, ConfirmState) -> + case gb_trees:is_empty(ConfirmState) of + true -> Collected; + false -> receive + #'basic.ack'{delivery_tag = DeliveryTag, + multiple = IsMulti} -> + Collected1 = case Collected of + none -> acks; + acks -> acks; + nacks -> mix; + mix -> mix + end, + drain_confirms(Collected1, + delete_confirms(DeliveryTag, IsMulti, + ConfirmState)); + #'basic.nack'{delivery_tag = DeliveryTag, + multiple = IsMulti} -> + Collected1 = case Collected of + none -> nacks; + nacks -> nacks; + acks -> mix; + mix -> mix + end, + drain_confirms(Collected1, + delete_confirms(DeliveryTag, IsMulti, + ConfirmState)) + after + 60000 -> {Collected, ConfirmState} + end + end. + +delete_confirms(DeliveryTag, false, ConfirmState) -> + gb_trees:delete(DeliveryTag, ConfirmState); +delete_confirms(DeliveryTag, true, ConfirmState) -> + multi_confirm(DeliveryTag, ConfirmState). + +multi_confirm(DeliveryTag, ConfirmState) -> + case gb_trees:is_empty(ConfirmState) of + true -> ConfirmState; + false -> {Key, _, ConfirmState1} = gb_trees:take_smallest(ConfirmState), + case Key =< DeliveryTag of + true -> multi_confirm(DeliveryTag, ConfirmState1); + false -> ConfirmState + end + end. diff --git a/deps/rabbit/test/rabbit_msg_record_SUITE.erl b/deps/rabbit/test/rabbit_msg_record_SUITE.erl new file mode 100644 index 0000000000..a82ba7481d --- /dev/null +++ b/deps/rabbit/test/rabbit_msg_record_SUITE.erl @@ -0,0 +1,213 @@ +-module(rabbit_msg_record_SUITE). + +-compile(export_all). + +-export([ + ]). + +-include("rabbit.hrl"). +-include("rabbit_framing.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("amqp10_common/include/amqp10_framing.hrl"). + +%%%=================================================================== +%%% Common Test callbacks +%%%=================================================================== + +all() -> + [ + {group, tests} + ]. + + +all_tests() -> + [ + ampq091_roundtrip, + message_id_ulong, + message_id_uuid, + message_id_binary, + message_id_large_binary, + message_id_large_string + ]. + +groups() -> + [ + {tests, [], all_tests()} + ]. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_Group, Config) -> + Config. + +end_per_group(_Group, _Config) -> + ok. + +init_per_testcase(_TestCase, Config) -> + Config. + +end_per_testcase(_TestCase, _Config) -> + ok. + +%%%=================================================================== +%%% Test cases +%%%=================================================================== + +ampq091_roundtrip(_Config) -> + Props = #'P_basic'{content_type = <<"text/plain">>, + content_encoding = <<"gzip">>, + headers = [{<<"x-stream-offset">>, long, 99}, + {<<"x-string">>, longstr, <<"a string">>}, + {<<"x-bool">>, bool, false}, + {<<"x-unsignedbyte">>, unsignedbyte, 1}, + {<<"x-unsignedshort">>, unsignedshort, 1}, + {<<"x-unsignedint">>, unsignedint, 1}, + {<<"x-signedint">>, signedint, 1}, + {<<"x-timestamp">>, timestamp, 1}, + {<<"x-double">>, double, 1.0}, + {<<"x-float">>, float, 1.0}, + {<<"x-binary">>, binary, <<"data">>} + ], + delivery_mode = 2, + priority = 99, + correlation_id = <<"corr">> , + reply_to = <<"reply-to">>, + expiration = <<"1">>, + message_id = <<"msg-id">>, + timestamp = 99, + type = <<"45">>, + user_id = <<"banana">>, + app_id = <<"rmq">> + % cluster_id = <<"adf">> + }, + Payload = [<<"data">>], + test_amqp091_roundtrip(Props, Payload), + test_amqp091_roundtrip(#'P_basic'{}, Payload), + ok. + +message_id_ulong(_Config) -> + Num = 9876789, + ULong = erlang:integer_to_binary(Num), + P = #'v1_0.properties'{message_id = {ulong, Num}, + correlation_id = {ulong, Num}}, + D = #'v1_0.data'{content = <<"data">>}, + Bin = [amqp10_framing:encode_bin(P), + amqp10_framing:encode_bin(D)], + R = rabbit_msg_record:init(iolist_to_binary(Bin)), + {Props, _} = rabbit_msg_record:to_amqp091(R), + ?assertMatch(#'P_basic'{message_id = ULong, + correlation_id = ULong, + headers = + [ + %% ordering shouldn't matter + {<<"x-correlation-id-type">>, longstr, <<"ulong">>}, + {<<"x-message-id-type">>, longstr, <<"ulong">>} + ]}, + Props), + ok. + +message_id_uuid(_Config) -> + %% fake a uuid + UUId = erlang:md5(term_to_binary(make_ref())), + TextUUId = rabbit_data_coercion:to_binary(rabbit_guid:to_string(UUId)), + P = #'v1_0.properties'{message_id = {uuid, UUId}, + correlation_id = {uuid, UUId}}, + D = #'v1_0.data'{content = <<"data">>}, + Bin = [amqp10_framing:encode_bin(P), + amqp10_framing:encode_bin(D)], + R = rabbit_msg_record:init(iolist_to_binary(Bin)), + {Props, _} = rabbit_msg_record:to_amqp091(R), + ?assertMatch(#'P_basic'{message_id = TextUUId, + correlation_id = TextUUId, + headers = + [ + %% ordering shouldn't matter + {<<"x-correlation-id-type">>, longstr, <<"uuid">>}, + {<<"x-message-id-type">>, longstr, <<"uuid">>} + ]}, + Props), + ok. + +message_id_binary(_Config) -> + %% fake a uuid + Orig = <<"asdfasdf">>, + Text = base64:encode(Orig), + P = #'v1_0.properties'{message_id = {binary, Orig}, + correlation_id = {binary, Orig}}, + D = #'v1_0.data'{content = <<"data">>}, + Bin = [amqp10_framing:encode_bin(P), + amqp10_framing:encode_bin(D)], + R = rabbit_msg_record:init(iolist_to_binary(Bin)), + {Props, _} = rabbit_msg_record:to_amqp091(R), + ?assertMatch(#'P_basic'{message_id = Text, + correlation_id = Text, + headers = + [ + %% ordering shouldn't matter + {<<"x-correlation-id-type">>, longstr, <<"binary">>}, + {<<"x-message-id-type">>, longstr, <<"binary">>} + ]}, + Props), + ok. + +message_id_large_binary(_Config) -> + %% cannot fit in a shortstr + Orig = crypto:strong_rand_bytes(500), + P = #'v1_0.properties'{message_id = {binary, Orig}, + correlation_id = {binary, Orig}}, + D = #'v1_0.data'{content = <<"data">>}, + Bin = [amqp10_framing:encode_bin(P), + amqp10_framing:encode_bin(D)], + R = rabbit_msg_record:init(iolist_to_binary(Bin)), + {Props, _} = rabbit_msg_record:to_amqp091(R), + ?assertMatch(#'P_basic'{message_id = undefined, + correlation_id = undefined, + headers = + [ + %% ordering shouldn't matter + {<<"x-correlation-id">>, longstr, Orig}, + {<<"x-message-id">>, longstr, Orig} + ]}, + Props), + ok. + +message_id_large_string(_Config) -> + %% cannot fit in a shortstr + Orig = base64:encode(crypto:strong_rand_bytes(500)), + P = #'v1_0.properties'{message_id = {utf8, Orig}, + correlation_id = {utf8, Orig}}, + D = #'v1_0.data'{content = <<"data">>}, + Bin = [amqp10_framing:encode_bin(P), + amqp10_framing:encode_bin(D)], + R = rabbit_msg_record:init(iolist_to_binary(Bin)), + {Props, _} = rabbit_msg_record:to_amqp091(R), + ?assertMatch(#'P_basic'{message_id = undefined, + correlation_id = undefined, + headers = + [ + %% ordering shouldn't matter + {<<"x-correlation-id">>, longstr, Orig}, + {<<"x-message-id">>, longstr, Orig} + ]}, + Props), + ok. + +%% Utility + +test_amqp091_roundtrip(Props, Payload) -> + MsgRecord0 = rabbit_msg_record:from_amqp091(Props, Payload), + MsgRecord = rabbit_msg_record:init( + iolist_to_binary(rabbit_msg_record:to_iodata(MsgRecord0))), + % meck:unload(), + {PropsOut, PayloadOut} = rabbit_msg_record:to_amqp091(MsgRecord), + ?assertEqual(Props, PropsOut), + ?assertEqual(iolist_to_binary(Payload), + iolist_to_binary(PayloadOut)), + ok. + + diff --git a/deps/rabbit/test/rabbit_stream_queue_SUITE.erl b/deps/rabbit/test/rabbit_stream_queue_SUITE.erl new file mode 100644 index 0000000000..a1055458db --- /dev/null +++ b/deps/rabbit/test/rabbit_stream_queue_SUITE.erl @@ -0,0 +1,1610 @@ +%% The contents of this file are subject to the Mozilla Public License +%% Version 1.1 (the "License"); you may not use this file except in +%% compliance with the License. You may obtain a copy of the License +%% at https://www.mozilla.org/MPL/ +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and +%% limitations under the License. +%% +%% The Original Code is RabbitMQ. +%% +%% Copyright (c) 2012-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(rabbit_stream_queue_SUITE). + +-include_lib("proper/include/proper.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). + +-compile(export_all). + +suite() -> + [{timetrap, 5 * 60000}]. + +all() -> + [ + {group, single_node}, + {group, cluster_size_2}, + {group, cluster_size_3}, + {group, unclustered_size_3_1}, + {group, unclustered_size_3_2}, + {group, unclustered_size_3_3}, + {group, cluster_size_3_1} + ]. + +groups() -> + [ + {single_node, [], [restart_single_node] ++ all_tests()}, + {cluster_size_2, [], all_tests()}, + {cluster_size_3, [], all_tests() ++ + [delete_replica, + delete_down_replica, + delete_classic_replica, + delete_quorum_replica, + consume_from_replica, + leader_failover, + initial_cluster_size_one, + initial_cluster_size_two, + initial_cluster_size_one_policy, + leader_locator_client_local, + leader_locator_random, + leader_locator_least_leaders, + leader_locator_policy]}, + {unclustered_size_3_1, [], [add_replica]}, + {unclustered_size_3_2, [], [consume_without_local_replica]}, + {unclustered_size_3_3, [], [grow_coordinator_cluster]}, + {cluster_size_3_1, [], [shrink_coordinator_cluster]} + ]. + +all_tests() -> + [ + declare_args, + declare_max_age, + declare_invalid_properties, + declare_server_named, + declare_queue, + delete_queue, + publish, + publish_confirm, + recover, + consume_without_qos, + consume, + consume_offset, + consume_timestamp_offset, + consume_timestamp_last_offset, + basic_get, + consume_with_autoack, + consume_and_nack, + consume_and_ack, + consume_and_reject, + consume_from_last, + consume_from_next, + consume_from_default, + consume_credit, + consume_credit_out_of_order_ack, + consume_credit_multiple_ack, + basic_cancel, + max_length_bytes, + max_age, + invalid_policy, + max_age_policy, + max_segment_size_policy, + purge + ]. + +%% ------------------------------------------------------------------- +%% Testsuite setup/teardown. +%% ------------------------------------------------------------------- + +init_per_suite(Config0) -> + rabbit_ct_helpers:log_environment(), + Config = rabbit_ct_helpers:merge_app_env( + Config0, {rabbit, [{stream_tick_interval, 1000}, + {log, [{file, [{level, debug}]}]}]}), + rabbit_ct_helpers:run_setup_steps(Config). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(Group, Config) -> + ClusterSize = case Group of + single_node -> 1; + cluster_size_2 -> 2; + cluster_size_3 -> 3; + cluster_size_3_1 -> 3; + unclustered_size_3_1 -> 3; + unclustered_size_3_2 -> 3; + unclustered_size_3_3 -> 3 + end, + Clustered = case Group of + unclustered_size_3_1 -> false; + unclustered_size_3_2 -> false; + unclustered_size_3_3 -> false; + _ -> true + end, + Config1 = rabbit_ct_helpers:set_config(Config, + [{rmq_nodes_count, ClusterSize}, + {rmq_nodename_suffix, Group}, + {tcp_ports_base}, + {rmq_nodes_clustered, Clustered}]), + Config1b = rabbit_ct_helpers:set_config(Config1, [{net_ticktime, 10}]), + Ret = rabbit_ct_helpers:run_steps(Config1b, + [fun merge_app_env/1 ] ++ + rabbit_ct_broker_helpers:setup_steps()), + case Ret of + {skip, _} -> + Ret; + Config2 -> + EnableFF = rabbit_ct_broker_helpers:enable_feature_flag( + Config2, stream_queue), + case EnableFF of + ok -> + ok = rabbit_ct_broker_helpers:rpc( + Config2, 0, application, set_env, + [rabbit, channel_tick_interval, 100]), + Config2; + Skip -> + end_per_group(Group, Config2), + Skip + end + end. + +end_per_group(_, Config) -> + rabbit_ct_helpers:run_steps(Config, + rabbit_ct_broker_helpers:teardown_steps()). + +init_per_testcase(Testcase, Config) -> + Config1 = rabbit_ct_helpers:testcase_started(Config, Testcase), + Q = rabbit_data_coercion:to_binary(Testcase), + Config2 = rabbit_ct_helpers:set_config(Config1, [{queue_name, Q}]), + rabbit_ct_helpers:run_steps(Config2, rabbit_ct_client_helpers:setup_steps()). + +merge_app_env(Config) -> + rabbit_ct_helpers:merge_app_env(Config, + {rabbit, [{core_metrics_gc_interval, 100}]}). + +end_per_testcase(Testcase, Config) -> + rabbit_ct_broker_helpers:rpc(Config, 0, ?MODULE, delete_queues, []), + Config1 = rabbit_ct_helpers:run_steps( + Config, + rabbit_ct_client_helpers:teardown_steps()), + rabbit_ct_helpers:testcase_finished(Config1, Testcase). + +%% ------------------------------------------------------------------- +%% Testcases. +%% ------------------------------------------------------------------- + +declare_args(Config) -> + Server = rabbit_ct_broker_helpers:get_node_config(Config, 0, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + Q = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', Q, 0, 0}, + declare(Ch, Q, [{<<"x-queue-type">>, longstr, <<"stream">>}, + {<<"x-max-length">>, long, 2000}])), + assert_queue_type(Server, Q, rabbit_stream_queue). + +declare_max_age(Config) -> + Server = rabbit_ct_broker_helpers:get_node_config(Config, 0, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + Q = ?config(queue_name, Config), + + ?assertExit( + {{shutdown, {server_initiated_close, 406, _}}, _}, + declare(rabbit_ct_client_helpers:open_channel(Config, Server), Q, + [{<<"x-queue-type">>, longstr, <<"stream">>}, + {<<"x-max-age">>, longstr, <<"1A">>}])), + + ?assertEqual({'queue.declare_ok', Q, 0, 0}, + declare(Ch, Q, [{<<"x-queue-type">>, longstr, <<"stream">>}, + {<<"x-max-age">>, longstr, <<"1Y">>}])), + assert_queue_type(Server, Q, rabbit_stream_queue). + +declare_invalid_properties(Config) -> + Server = rabbit_ct_broker_helpers:get_node_config(Config, 0, nodename), + Q = ?config(queue_name, Config), + + ?assertExit( + {{shutdown, {server_initiated_close, 406, _}}, _}, + amqp_channel:call( + rabbit_ct_client_helpers:open_channel(Config, Server), + #'queue.declare'{queue = Q, + auto_delete = true, + durable = true, + arguments = [{<<"x-queue-type">>, longstr, <<"stream">>}]})), + ?assertExit( + {{shutdown, {server_initiated_close, 406, _}}, _}, + amqp_channel:call( + rabbit_ct_client_helpers:open_channel(Config, Server), + #'queue.declare'{queue = Q, + exclusive = true, + durable = true, + arguments = [{<<"x-queue-type">>, longstr, <<"stream">>}]})), + ?assertExit( + {{shutdown, {server_initiated_close, 406, _}}, _}, + amqp_channel:call( + rabbit_ct_client_helpers:open_channel(Config, Server), + #'queue.declare'{queue = Q, + durable = false, + arguments = [{<<"x-queue-type">>, longstr, <<"stream">>}]})). + +declare_server_named(Config) -> + Server = rabbit_ct_broker_helpers:get_node_config(Config, 0, nodename), + + ?assertExit( + {{shutdown, {server_initiated_close, 406, _}}, _}, + declare(rabbit_ct_client_helpers:open_channel(Config, Server), + <<"">>, [{<<"x-queue-type">>, longstr, <<"stream">>}])). + +declare_queue(Config) -> + [Server | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + Q = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', Q, 0, 0}, + declare(Ch, Q, [{<<"x-queue-type">>, longstr, <<"stream">>}])), + + %% Test declare an existing queue + ?assertEqual({'queue.declare_ok', Q, 0, 0}, + declare(Ch, Q, [{<<"x-queue-type">>, longstr, <<"stream">>}])), + + ?assertMatch([_], rpc:call(Server, supervisor, which_children, + [osiris_server_sup])), + + %% Test declare an existing queue with different arguments + ?assertExit(_, declare(Ch, Q, [])). + +delete_queue(Config) -> + [Server | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + Q = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', Q, 0, 0}, + declare(Ch, Q, [{<<"x-queue-type">>, longstr, <<"stream">>}])), + ?assertMatch(#'queue.delete_ok'{}, + amqp_channel:call(Ch, #'queue.delete'{queue = Q})). + +add_replica(Config) -> + [Server0, Server1, Server2] = + rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + Ch = rabbit_ct_client_helpers:open_channel(Config, Server0), + Q = ?config(queue_name, Config), + + %% Let's also try the add replica command on other queue types, it should fail + %% We're doing it in the same test for efficiency, otherwise we have to + %% start new rabbitmq clusters every time for a minor testcase + QClassic = <<Q/binary, "_classic">>, + QQuorum = <<Q/binary, "_quorum">>, + + ?assertEqual({'queue.declare_ok', Q, 0, 0}, + declare(Ch, Q, [{<<"x-queue-type">>, longstr, <<"stream">>}])), + ?assertEqual({'queue.declare_ok', QClassic, 0, 0}, + declare(Ch, QClassic, [{<<"x-queue-type">>, longstr, <<"classic">>}])), + ?assertEqual({'queue.declare_ok', QQuorum, 0, 0}, + declare(Ch, QQuorum, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), + + %% Not a member of the cluster, what would happen? + ?assertEqual({error, node_not_running}, + rpc:call(Server0, rabbit_stream_queue, add_replica, + [<<"/">>, Q, Server1])), + ?assertEqual({error, classic_queue_not_supported}, + rpc:call(Server0, rabbit_stream_queue, add_replica, + [<<"/">>, QClassic, Server1])), + ?assertEqual({error, quorum_queue_not_supported}, + rpc:call(Server0, rabbit_stream_queue, add_replica, + [<<"/">>, QQuorum, Server1])), + + ok = rabbit_control_helper:command(stop_app, Server1), + ok = rabbit_control_helper:command(join_cluster, Server1, [atom_to_list(Server0)], []), + rabbit_control_helper:command(start_app, Server1), + timer:sleep(1000), + ?assertEqual({error, classic_queue_not_supported}, + rpc:call(Server0, rabbit_stream_queue, add_replica, + [<<"/">>, QClassic, Server1])), + ?assertEqual({error, quorum_queue_not_supported}, + rpc:call(Server0, rabbit_stream_queue, add_replica, + [<<"/">>, QQuorum, Server1])), + ?assertEqual(ok, + rpc:call(Server0, rabbit_stream_queue, add_replica, + [<<"/">>, Q, Server1])), + %% replicas must be recorded on the state, and if we publish messages then they must + %% be stored on disk + check_leader_and_replicas(Config, Q, Server0, [Server1]), + %% And if we try again? Idempotent + ?assertEqual(ok, rpc:call(Server0, rabbit_stream_queue, add_replica, + [<<"/">>, Q, Server1])), + %% Add another node + ok = rabbit_control_helper:command(stop_app, Server2), + ok = rabbit_control_helper:command(join_cluster, Server2, [atom_to_list(Server0)], []), + rabbit_control_helper:command(start_app, Server2), + ?assertEqual(ok, rpc:call(Server0, rabbit_stream_queue, add_replica, + [<<"/">>, Q, Server2])), + check_leader_and_replicas(Config, Q, Server0, [Server1, Server2]). + +delete_replica(Config) -> + [Server0, Server1, Server2] = + rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + Ch = rabbit_ct_client_helpers:open_channel(Config, Server0), + Q = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', Q, 0, 0}, + declare(Ch, Q, [{<<"x-queue-type">>, longstr, <<"stream">>}])), + check_leader_and_replicas(Config, Q, Server0, [Server1, Server2]), + %% Not a member of the cluster, what would happen? + ?assertEqual({error, node_not_running}, + rpc:call(Server0, rabbit_stream_queue, delete_replica, + [<<"/">>, Q, 'zen@rabbit'])), + ?assertEqual(ok, + rpc:call(Server0, rabbit_stream_queue, delete_replica, + [<<"/">>, Q, Server1])), + %% check it's gone + check_leader_and_replicas(Config, Q, Server0, [Server2]), + %% And if we try again? Idempotent + ?assertEqual(ok, rpc:call(Server0, rabbit_stream_queue, delete_replica, + [<<"/">>, Q, Server1])), + %% Delete the last replica + ?assertEqual(ok, rpc:call(Server0, rabbit_stream_queue, delete_replica, + [<<"/">>, Q, Server2])), + check_leader_and_replicas(Config, Q, Server0, []). + +grow_coordinator_cluster(Config) -> + [Server0, Server1, _Server2] = + rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + Ch = rabbit_ct_client_helpers:open_channel(Config, Server0), + Q = ?config(queue_name, Config), + + ?assertEqual({'queue.declare_ok', Q, 0, 0}, + declare(Ch, Q, [{<<"x-queue-type">>, longstr, <<"stream">>}])), + + ok = rabbit_control_helper:command(stop_app, Server1), + ok = rabbit_control_helper:command(join_cluster, Server1, [atom_to_list(Server0)], []), + rabbit_control_helper:command(start_app, Server1), + + rabbit_ct_helpers:await_condition( + fun() -> + case rpc:call(Server0, ra, members, [{rabbit_stream_coordinator, Server0}]) of + {_, Members, _} -> + Nodes = lists:sort([N || {_, N} <- Members]), + lists:sort([Server0, Server1]) == Nodes; + _ -> + false + end + end, 60000). + +shrink_coordinator_cluster(Config) -> + [Server0, Server1, Server2] = + rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + Ch = rabbit_ct_client_helpers:open_channel(Config, Server0), + Q = ?config(queue_name, Config), + + ?assertEqual({'queue.declare_ok', Q, 0, 0}, + declare(Ch, Q, [{<<"x-queue-type">>, longstr, <<"stream">>}])), + + ok = rabbit_control_helper:command(stop_app, Server2), + ok = rabbit_control_helper:command(forget_cluster_node, Server0, [atom_to_list(Server2)], []), + + rabbit_ct_helpers:await_condition( + fun() -> + case rpc:call(Server0, ra, members, [{rabbit_stream_coordinator, Server0}]) of + {_, Members, _} -> + Nodes = lists:sort([N || {_, N} <- Members]), + lists:sort([Server0, Server1]) == Nodes; + _ -> + false + end + end, 60000). + +delete_classic_replica(Config) -> + [Server0, Server1, _Server2] = + rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + Ch = rabbit_ct_client_helpers:open_channel(Config, Server0), + Q = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', Q, 0, 0}, + declare(Ch, Q, [{<<"x-queue-type">>, longstr, <<"classic">>}])), + %% Not a member of the cluster, what would happen? + ?assertEqual({error, classic_queue_not_supported}, + rpc:call(Server0, rabbit_stream_queue, delete_replica, + [<<"/">>, Q, 'zen@rabbit'])), + ?assertEqual({error, classic_queue_not_supported}, + rpc:call(Server0, rabbit_stream_queue, delete_replica, + [<<"/">>, Q, Server1])). + +delete_quorum_replica(Config) -> + [Server0, Server1, _Server2] = + rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + Ch = rabbit_ct_client_helpers:open_channel(Config, Server0), + Q = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', Q, 0, 0}, + declare(Ch, Q, [{<<"x-queue-type">>, longstr, <<"quorum">>}])), + %% Not a member of the cluster, what would happen? + ?assertEqual({error, quorum_queue_not_supported}, + rpc:call(Server0, rabbit_stream_queue, delete_replica, + [<<"/">>, Q, 'zen@rabbit'])), + ?assertEqual({error, quorum_queue_not_supported}, + rpc:call(Server0, rabbit_stream_queue, delete_replica, + [<<"/">>, Q, Server1])). + +delete_down_replica(Config) -> + [Server0, Server1, Server2] = + rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + Ch = rabbit_ct_client_helpers:open_channel(Config, Server0), + Q = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', Q, 0, 0}, + declare(Ch, Q, [{<<"x-queue-type">>, longstr, <<"stream">>}])), + check_leader_and_replicas(Config, Q, Server0, [Server1, Server2]), + ok = rabbit_ct_broker_helpers:stop_node(Config, Server1), + ?assertEqual({error, node_not_running}, + rpc:call(Server0, rabbit_stream_queue, delete_replica, + [<<"/">>, Q, Server1])), + %% check it isn't gone + check_leader_and_replicas(Config, Q, Server0, [Server1, Server2]), + ok = rabbit_ct_broker_helpers:start_node(Config, Server1), + ?assertEqual(ok, + rpc:call(Server0, rabbit_stream_queue, delete_replica, + [<<"/">>, Q, Server1])). + +publish(Config) -> + [Server | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + Q = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', Q, 0, 0}, + declare(Ch, Q, [{<<"x-queue-type">>, longstr, <<"stream">>}])), + + publish(Ch, Q), + quorum_queue_utils:wait_for_messages(Config, [[Q, <<"1">>, <<"1">>, <<"0">>]]). + +publish_confirm(Config) -> + [Server | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + Q = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', Q, 0, 0}, + declare(Ch, Q, [{<<"x-queue-type">>, longstr, <<"stream">>}])), + + #'confirm.select_ok'{} = amqp_channel:call(Ch, #'confirm.select'{}), + amqp_channel:register_confirm_handler(Ch, self()), + publish(Ch, Q), + amqp_channel:wait_for_confirms(Ch, 5), + quorum_queue_utils:wait_for_messages(Config, [[Q, <<"1">>, <<"1">>, <<"0">>]]). + +restart_single_node(Config) -> + [Server] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + Q = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', Q, 0, 0}, + declare(Ch, Q, [{<<"x-queue-type">>, longstr, <<"stream">>}])), + publish(Ch, Q), + quorum_queue_utils:wait_for_messages(Config, [[Q, <<"1">>, <<"1">>, <<"0">>]]), + + rabbit_control_helper:command(stop_app, Server), + rabbit_control_helper:command(start_app, Server), + + quorum_queue_utils:wait_for_messages(Config, [[Q, <<"1">>, <<"1">>, <<"0">>]]), + Ch1 = rabbit_ct_client_helpers:open_channel(Config, Server), + publish(Ch1, Q), + quorum_queue_utils:wait_for_messages(Config, [[Q, <<"2">>, <<"2">>, <<"0">>]]). + +recover(Config) -> + [Server | _] = Servers = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + Q = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', Q, 0, 0}, + declare(Ch, Q, [{<<"x-queue-type">>, longstr, <<"stream">>}])), + publish(Ch, Q), + quorum_queue_utils:wait_for_messages(Config, [[Q, <<"1">>, <<"1">>, <<"0">>]]), + + [rabbit_ct_broker_helpers:stop_node(Config, S) || S <- Servers], + [rabbit_ct_broker_helpers:start_node(Config, S) || S <- lists:reverse(Servers)], + + quorum_queue_utils:wait_for_messages(Config, [[Q, <<"1">>, <<"1">>, <<"0">>]]), + Ch1 = rabbit_ct_client_helpers:open_channel(Config, Server), + publish(Ch1, Q), + quorum_queue_utils:wait_for_messages(Config, [[Q, <<"2">>, <<"2">>, <<"0">>]]). + +consume_without_qos(Config) -> + [Server | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + Q = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', Q, 0, 0}, + declare(Ch, Q, [{<<"x-queue-type">>, longstr, <<"stream">>}])), + + ?assertExit({{shutdown, {server_initiated_close, 406, _}}, _}, + amqp_channel:subscribe(Ch, #'basic.consume'{queue = Q, consumer_tag = <<"ctag">>}, + self())). + +consume_without_local_replica(Config) -> + [Server0, Server1 | _] = + rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + Ch = rabbit_ct_client_helpers:open_channel(Config, Server0), + Q = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', Q, 0, 0}, + declare(Ch, Q, [{<<"x-queue-type">>, longstr, <<"stream">>}])), + %% Add another node to the cluster, but it won't have a replica + ok = rabbit_control_helper:command(stop_app, Server1), + ok = rabbit_control_helper:command(join_cluster, Server1, [atom_to_list(Server0)], []), + rabbit_control_helper:command(start_app, Server1), + timer:sleep(1000), + + Ch1 = rabbit_ct_client_helpers:open_channel(Config, Server1), + qos(Ch1, 10, false), + ?assertExit({{shutdown, {server_initiated_close, 406, _}}, _}, + amqp_channel:subscribe(Ch1, #'basic.consume'{queue = Q, consumer_tag = <<"ctag">>}, + self())). + +consume(Config) -> + [Server | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + Q = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', Q, 0, 0}, + declare(Ch, Q, [{<<"x-queue-type">>, longstr, <<"stream">>}])), + + #'confirm.select_ok'{} = amqp_channel:call(Ch, #'confirm.select'{}), + amqp_channel:register_confirm_handler(Ch, self()), + publish(Ch, Q), + amqp_channel:wait_for_confirms(Ch, 5), + + Ch1 = rabbit_ct_client_helpers:open_channel(Config, Server), + qos(Ch1, 10, false), + subscribe(Ch1, Q, false, 0), + receive + {#'basic.deliver'{delivery_tag = DeliveryTag}, _} -> + ok = amqp_channel:cast(Ch1, #'basic.ack'{delivery_tag = DeliveryTag, + multiple = false}), + _ = amqp_channel:call(Ch1, #'basic.cancel'{consumer_tag = <<"ctag">>}), + ok = amqp_channel:close(Ch1), + ok + after 5000 -> + exit(timeout) + end. + +consume_offset(Config) -> + [Server | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + Q = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', Q, 0, 0}, + declare(Ch, Q, [{<<"x-queue-type">>, longstr, <<"stream">>}])), + + #'confirm.select_ok'{} = amqp_channel:call(Ch, #'confirm.select'{}), + amqp_channel:register_confirm_handler(Ch, self()), + Payload = << <<"1">> || _ <- lists:seq(1, 500) >>, + [publish(Ch, Q, Payload) || _ <- lists:seq(1, 1000)], + amqp_channel:wait_for_confirms(Ch, 5), + + run_proper( + fun () -> + ?FORALL(Offset, range(0, 999), + begin + Ch1 = rabbit_ct_client_helpers:open_channel(Config, Server), + qos(Ch1, 10, false), + subscribe(Ch1, Q, false, Offset), + receive_batch(Ch1, Offset, 999), + receive + {_, + #amqp_msg{props = #'P_basic'{headers = [{<<"x-stream-offset">>, long, S}]}}} + when S < Offset -> + exit({unexpected_offset, S}) + after 1000 -> + ok + end, + amqp_channel:call(Ch1, #'basic.cancel'{consumer_tag = <<"ctag">>}), + true + end) + end, [], 25). + +consume_timestamp_offset(Config) -> + [Server | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + Q = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', Q, 0, 0}, + declare(Ch, Q, [{<<"x-queue-type">>, longstr, <<"stream">>}])), + + #'confirm.select_ok'{} = amqp_channel:call(Ch, #'confirm.select'{}), + amqp_channel:register_confirm_handler(Ch, self()), + Payload = <<"111">>, + [publish(Ch, Q, Payload) || _ <- lists:seq(1, 100)], + amqp_channel:wait_for_confirms(Ch, 5), + + + Ch1 = rabbit_ct_client_helpers:open_channel(Config, Server), + qos(Ch1, 10, false), + + Offset = erlang:system_time(millisecond) - 600000, + amqp_channel:subscribe( + Ch1, + #'basic.consume'{queue = Q, + no_ack = false, + consumer_tag = <<"ctag">>, + arguments = [{<<"x-stream-offset">>, timestamp, Offset}]}, + self()), + receive + #'basic.consume_ok'{consumer_tag = <<"ctag">>} -> + ok + end, + + %% It has subscribed to a very old timestamp, so we will receive the whole stream + receive_batch(Ch1, 0, 99). + +consume_timestamp_last_offset(Config) -> + [Server | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + Q = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', Q, 0, 0}, + declare(Ch, Q, [{<<"x-queue-type">>, longstr, <<"stream">>}])), + + #'confirm.select_ok'{} = amqp_channel:call(Ch, #'confirm.select'{}), + amqp_channel:register_confirm_handler(Ch, self()), + [publish(Ch, Q, <<"111">>) || _ <- lists:seq(1, 100)], + amqp_channel:wait_for_confirms(Ch, 5), + + + Ch1 = rabbit_ct_client_helpers:open_channel(Config, Server), + qos(Ch1, 10, false), + + %% Subscribe from now/future + Offset = erlang:system_time(millisecond) + 60000, + amqp_channel:subscribe( + Ch1, + #'basic.consume'{queue = Q, + no_ack = false, + consumer_tag = <<"ctag">>, + arguments = [{<<"x-stream-offset">>, timestamp, Offset}]}, + self()), + receive + #'basic.consume_ok'{consumer_tag = <<"ctag">>} -> + ok + end, + + receive + {_, + #amqp_msg{props = #'P_basic'{headers = [{<<"x-stream-offset">>, long, S}]}}} + when S < 100 -> + exit({unexpected_offset, S}) + after 1000 -> + ok + end, + + %% Publish a few more + [publish(Ch, Q, <<"msg2">>) || _ <- lists:seq(1, 100)], + amqp_channel:wait_for_confirms(Ch, 5), + + %% Yeah! we got them + receive_batch(Ch1, 100, 199). + +basic_get(Config) -> + [Server | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + Q = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', Q, 0, 0}, + declare(Ch, Q, [{<<"x-queue-type">>, longstr, <<"stream">>}])), + + ?assertExit({{shutdown, {connection_closing, {server_initiated_close, 540, _}}}, _}, + amqp_channel:call(Ch, #'basic.get'{queue = Q})). + +consume_with_autoack(Config) -> + [Server | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + Q = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', Q, 0, 0}, + declare(Ch, Q, [{<<"x-queue-type">>, longstr, <<"stream">>}])), + + Ch1 = rabbit_ct_client_helpers:open_channel(Config, Server), + qos(Ch1, 10, false), + + ?assertExit( + {{shutdown, {connection_closing, {server_initiated_close, 540, _}}}, _}, + subscribe(Ch1, Q, true, 0)). + +consume_and_nack(Config) -> + [Server | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + Q = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', Q, 0, 0}, + declare(Ch, Q, [{<<"x-queue-type">>, longstr, <<"stream">>}])), + + #'confirm.select_ok'{} = amqp_channel:call(Ch, #'confirm.select'{}), + amqp_channel:register_confirm_handler(Ch, self()), + publish(Ch, Q), + amqp_channel:wait_for_confirms(Ch, 5), + + Ch1 = rabbit_ct_client_helpers:open_channel(Config, Server), + qos(Ch1, 10, false), + subscribe(Ch1, Q, false, 0), + receive + {#'basic.deliver'{delivery_tag = DeliveryTag}, _} -> + ok = amqp_channel:cast(Ch1, #'basic.nack'{delivery_tag = DeliveryTag, + multiple = false, + requeue = true}), + %% Nack will throw a not implemented exception. As it is a cast operation, + %% we'll detect the conneciton/channel closure on the next call. + %% Let's try to redeclare and see what happens + ?assertExit({{shutdown, {connection_closing, {server_initiated_close, 540, _}}}, _}, + declare(Ch1, Q, [{<<"x-queue-type">>, longstr, <<"stream">>}])) + after 10000 -> + exit(timeout) + end. + +basic_cancel(Config) -> + [Server | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + Q = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', Q, 0, 0}, + declare(Ch, Q, [{<<"x-queue-type">>, longstr, <<"stream">>}])), + + #'confirm.select_ok'{} = amqp_channel:call(Ch, #'confirm.select'{}), + amqp_channel:register_confirm_handler(Ch, self()), + publish(Ch, Q), + amqp_channel:wait_for_confirms(Ch, 5), + + Ch1 = rabbit_ct_client_helpers:open_channel(Config, Server), + qos(Ch1, 10, false), + subscribe(Ch1, Q, false, 0), + rabbit_ct_helpers:await_condition( + fun() -> + 1 == length(rabbit_ct_broker_helpers:rpc(Config, Server, ets, tab2list, + [consumer_created])) + end, 30000), + receive + {#'basic.deliver'{}, _} -> + amqp_channel:call(Ch1, #'basic.cancel'{consumer_tag = <<"ctag">>}), + ?assertMatch([], rabbit_ct_broker_helpers:rpc(Config, Server, ets, tab2list, [consumer_created])) + after 10000 -> + exit(timeout) + end. + +consume_and_reject(Config) -> + [Server | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + Q = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', Q, 0, 0}, + declare(Ch, Q, [{<<"x-queue-type">>, longstr, <<"stream">>}])), + + #'confirm.select_ok'{} = amqp_channel:call(Ch, #'confirm.select'{}), + amqp_channel:register_confirm_handler(Ch, self()), + publish(Ch, Q), + amqp_channel:wait_for_confirms(Ch, 5), + + Ch1 = rabbit_ct_client_helpers:open_channel(Config, Server), + qos(Ch1, 10, false), + subscribe(Ch1, Q, false, 0), + receive + {#'basic.deliver'{delivery_tag = DeliveryTag}, _} -> + ok = amqp_channel:cast(Ch1, #'basic.reject'{delivery_tag = DeliveryTag, + requeue = true}), + %% Reject will throw a not implemented exception. As it is a cast operation, + %% we'll detect the conneciton/channel closure on the next call. + %% Let's try to redeclare and see what happens + ?assertExit({{shutdown, {connection_closing, {server_initiated_close, 540, _}}}, _}, + declare(Ch1, Q, [{<<"x-queue-type">>, longstr, <<"stream">>}])) + after 10000 -> + exit(timeout) + end. + +consume_and_ack(Config) -> + [Server | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + Q = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', Q, 0, 0}, + declare(Ch, Q, [{<<"x-queue-type">>, longstr, <<"stream">>}])), + + #'confirm.select_ok'{} = amqp_channel:call(Ch, #'confirm.select'{}), + amqp_channel:register_confirm_handler(Ch, self()), + publish(Ch, Q), + amqp_channel:wait_for_confirms(Ch, 5), + + Ch1 = rabbit_ct_client_helpers:open_channel(Config, Server), + qos(Ch1, 10, false), + subscribe(Ch1, Q, false, 0), + receive + {#'basic.deliver'{delivery_tag = DeliveryTag}, _} -> + ok = amqp_channel:cast(Ch1, #'basic.ack'{delivery_tag = DeliveryTag, + multiple = false}), + %% It will succeed as ack is now a credit operation. We should be + %% able to redeclare a queue (gen_server call op) as the channel + %% should still be open and declare is an idempotent operation + ?assertEqual({'queue.declare_ok', Q, 0, 0}, + declare(Ch1, Q, [{<<"x-queue-type">>, longstr, <<"stream">>}])), + quorum_queue_utils:wait_for_messages(Config, [[Q, <<"1">>, <<"1">>, <<"0">>]]) + after 5000 -> + exit(timeout) + end. + +consume_from_last(Config) -> + [Server | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + Q = ?config(queue_name, Config), + + ?assertEqual({'queue.declare_ok', Q, 0, 0}, + declare(Ch, Q, [{<<"x-queue-type">>, longstr, <<"stream">>}])), + + #'confirm.select_ok'{} = amqp_channel:call(Ch, #'confirm.select'{}), + amqp_channel:register_confirm_handler(Ch, self()), + [publish(Ch, Q, <<"msg1">>) || _ <- lists:seq(1, 100)], + amqp_channel:wait_for_confirms(Ch, 5), + + Ch1 = rabbit_ct_client_helpers:open_channel(Config, Server), + qos(Ch1, 10, false), + + [Info] = rabbit_ct_broker_helpers:rpc(Config, 0, rabbit_amqqueue, + info_all, [<<"/">>, [committed_offset]]), + + %% We'll receive data from the last committed offset, let's check that is not the + %% first offset + CommittedOffset = proplists:get_value(committed_offset, Info), + ?assert(CommittedOffset > 0), + + %% If the offset is not provided, we're subscribing to the tail of the stream + amqp_channel:subscribe( + Ch1, #'basic.consume'{queue = Q, + no_ack = false, + consumer_tag = <<"ctag">>, + arguments = [{<<"x-stream-offset">>, longstr, <<"last">>}]}, + self()), + receive + #'basic.consume_ok'{consumer_tag = <<"ctag">>} -> + ok + end, + + %% And receive the messages from the last committed offset to the end of the stream + receive_batch(Ch1, CommittedOffset, 99), + + %% Publish a few more + [publish(Ch, Q, <<"msg2">>) || _ <- lists:seq(1, 100)], + amqp_channel:wait_for_confirms(Ch, 5), + + %% Yeah! we got them + receive_batch(Ch1, 100, 199). + +consume_from_next(Config) -> + consume_from_next(Config, [{<<"x-stream-offset">>, longstr, <<"next">>}]). + +consume_from_default(Config) -> + consume_from_next(Config, []). + +consume_from_next(Config, Args) -> + [Server | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + Q = ?config(queue_name, Config), + + ?assertEqual({'queue.declare_ok', Q, 0, 0}, + declare(Ch, Q, [{<<"x-queue-type">>, longstr, <<"stream">>}])), + + #'confirm.select_ok'{} = amqp_channel:call(Ch, #'confirm.select'{}), + amqp_channel:register_confirm_handler(Ch, self()), + [publish(Ch, Q, <<"msg1">>) || _ <- lists:seq(1, 100)], + amqp_channel:wait_for_confirms(Ch, 5), + + Ch1 = rabbit_ct_client_helpers:open_channel(Config, Server), + qos(Ch1, 10, false), + + [Info] = rabbit_ct_broker_helpers:rpc(Config, 0, rabbit_amqqueue, + info_all, [<<"/">>, [committed_offset]]), + + %% We'll receive data from the last committed offset, let's check that is not the + %% first offset + CommittedOffset = proplists:get_value(committed_offset, Info), + ?assert(CommittedOffset > 0), + + %% If the offset is not provided, we're subscribing to the tail of the stream + amqp_channel:subscribe( + Ch1, #'basic.consume'{queue = Q, + no_ack = false, + consumer_tag = <<"ctag">>, + arguments = Args}, + self()), + receive + #'basic.consume_ok'{consumer_tag = <<"ctag">>} -> + ok + end, + + %% Publish a few more + [publish(Ch, Q, <<"msg2">>) || _ <- lists:seq(1, 100)], + amqp_channel:wait_for_confirms(Ch, 5), + + %% Yeah! we got them + receive_batch(Ch1, 100, 199). + +consume_from_replica(Config) -> + [Server1, Server2 | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch1 = rabbit_ct_client_helpers:open_channel(Config, Server1), + Q = ?config(queue_name, Config), + + ?assertEqual({'queue.declare_ok', Q, 0, 0}, + declare(Ch1, Q, [{<<"x-queue-type">>, longstr, <<"stream">>}])), + + #'confirm.select_ok'{} = amqp_channel:call(Ch1, #'confirm.select'{}), + amqp_channel:register_confirm_handler(Ch1, self()), + [publish(Ch1, Q, <<"msg1">>) || _ <- lists:seq(1, 100)], + amqp_channel:wait_for_confirms(Ch1, 5), + + Ch2 = rabbit_ct_client_helpers:open_channel(Config, Server2), + qos(Ch2, 10, false), + + subscribe(Ch2, Q, false, 0), + receive_batch(Ch2, 0, 99). + +consume_credit(Config) -> + %% Because osiris provides one chunk on every read and we don't want to buffer + %% messages in the broker to avoid memory penalties, the credit value won't + %% be strict - we allow it into the negative values. + %% We can test that after receiving a chunk, no more messages are delivered until + %% the credit goes back to a positive value. + [Server | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + Q = ?config(queue_name, Config), + + ?assertEqual({'queue.declare_ok', Q, 0, 0}, + declare(Ch, Q, [{<<"x-queue-type">>, longstr, <<"stream">>}])), + + #'confirm.select_ok'{} = amqp_channel:call(Ch, #'confirm.select'{}), + amqp_channel:register_confirm_handler(Ch, self()), + %% Let's publish a big batch, to ensure we have more than a chunk available + NumMsgs = 100, + [publish(Ch, Q, <<"msg1">>) || _ <- lists:seq(1, NumMsgs)], + amqp_channel:wait_for_confirms(Ch, 5), + + Ch1 = rabbit_ct_client_helpers:open_channel(Config, Server), + + %% Let's subscribe with a small credit, easier to test + Credit = 2, + qos(Ch1, Credit, false), + subscribe(Ch1, Q, false, 0), + + %% Receive everything + DeliveryTags = receive_batch(), + + %% We receive at least the given credit as we know there are 100 messages in the queue + ?assert(length(DeliveryTags) >= Credit), + + %% Let's ack as many messages as we can while avoiding a positive credit for new deliveries + {ToAck, Pending} = lists:split(length(DeliveryTags) - Credit, DeliveryTags), + + [ok = amqp_channel:cast(Ch1, #'basic.ack'{delivery_tag = DeliveryTag, + multiple = false}) + || DeliveryTag <- ToAck], + + %% Nothing here, this is good + receive + {#'basic.deliver'{}, _} -> + exit(unexpected_delivery) + after 1000 -> + ok + end, + + %% Let's ack one more, we should receive a new chunk + ok = amqp_channel:cast(Ch1, #'basic.ack'{delivery_tag = hd(Pending), + multiple = false}), + + %% Yeah, here is the new chunk! + receive + {#'basic.deliver'{}, _} -> + ok + after 5000 -> + exit(timeout) + end. + +consume_credit_out_of_order_ack(Config) -> + %% Like consume_credit but acknowledging the messages out of order. + %% We want to ensure it doesn't behave like multiple, that is if we have + %% credit 2 and received 10 messages, sending the ack for the message id + %% number 10 should only increase credit by 1. + [Server | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + Q = ?config(queue_name, Config), + + ?assertEqual({'queue.declare_ok', Q, 0, 0}, + declare(Ch, Q, [{<<"x-queue-type">>, longstr, <<"stream">>}])), + + #'confirm.select_ok'{} = amqp_channel:call(Ch, #'confirm.select'{}), + amqp_channel:register_confirm_handler(Ch, self()), + %% Let's publish a big batch, to ensure we have more than a chunk available + NumMsgs = 100, + [publish(Ch, Q, <<"msg1">>) || _ <- lists:seq(1, NumMsgs)], + amqp_channel:wait_for_confirms(Ch, 5), + + Ch1 = rabbit_ct_client_helpers:open_channel(Config, Server), + + %% Let's subscribe with a small credit, easier to test + Credit = 2, + qos(Ch1, Credit, false), + subscribe(Ch1, Q, false, 0), + + %% ******* This is the difference with consume_credit + %% Receive everything, let's reverse the delivery tags here so we ack out of order + DeliveryTags = lists:reverse(receive_batch()), + + %% We receive at least the given credit as we know there are 100 messages in the queue + ?assert(length(DeliveryTags) >= Credit), + + %% Let's ack as many messages as we can while avoiding a positive credit for new deliveries + {ToAck, Pending} = lists:split(length(DeliveryTags) - Credit, DeliveryTags), + + [ok = amqp_channel:cast(Ch1, #'basic.ack'{delivery_tag = DeliveryTag, + multiple = false}) + || DeliveryTag <- ToAck], + + %% Nothing here, this is good + receive + {#'basic.deliver'{}, _} -> + exit(unexpected_delivery) + after 1000 -> + ok + end, + + %% Let's ack one more, we should receive a new chunk + ok = amqp_channel:cast(Ch1, #'basic.ack'{delivery_tag = hd(Pending), + multiple = false}), + + %% Yeah, here is the new chunk! + receive + {#'basic.deliver'{}, _} -> + ok + after 5000 -> + exit(timeout) + end. + +consume_credit_multiple_ack(Config) -> + %% Like consume_credit but acknowledging the messages out of order. + %% We want to ensure it doesn't behave like multiple, that is if we have + %% credit 2 and received 10 messages, sending the ack for the message id + %% number 10 should only increase credit by 1. + [Server | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + Q = ?config(queue_name, Config), + + ?assertEqual({'queue.declare_ok', Q, 0, 0}, + declare(Ch, Q, [{<<"x-queue-type">>, longstr, <<"stream">>}])), + + #'confirm.select_ok'{} = amqp_channel:call(Ch, #'confirm.select'{}), + amqp_channel:register_confirm_handler(Ch, self()), + %% Let's publish a big batch, to ensure we have more than a chunk available + NumMsgs = 100, + [publish(Ch, Q, <<"msg1">>) || _ <- lists:seq(1, NumMsgs)], + amqp_channel:wait_for_confirms(Ch, 5), + + Ch1 = rabbit_ct_client_helpers:open_channel(Config, Server), + + %% Let's subscribe with a small credit, easier to test + Credit = 2, + qos(Ch1, Credit, false), + subscribe(Ch1, Q, false, 0), + + %% ******* This is the difference with consume_credit + %% Receive everything, let's reverse the delivery tags here so we ack out of order + DeliveryTag = lists:last(receive_batch()), + + ok = amqp_channel:cast(Ch1, #'basic.ack'{delivery_tag = DeliveryTag, + multiple = true}), + + %% Yeah, here is the new chunk! + receive + {#'basic.deliver'{}, _} -> + ok + after 5000 -> + exit(timeout) + end. + +max_length_bytes(Config) -> + [Server | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + Q = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', Q, 0, 0}, + declare(Ch, Q, [{<<"x-queue-type">>, longstr, <<"stream">>}, + {<<"x-max-length-bytes">>, long, 500}, + {<<"x-max-segment-size">>, long, 250}])), + + Payload = << <<"1">> || _ <- lists:seq(1, 500) >>, + + #'confirm.select_ok'{} = amqp_channel:call(Ch, #'confirm.select'{}), + amqp_channel:register_confirm_handler(Ch, self()), + [publish(Ch, Q, Payload) || _ <- lists:seq(1, 100)], + amqp_channel:wait_for_confirms(Ch, 5), + + %% We don't yet have reliable metrics, as the committed offset doesn't work + %% as a counter once we start applying retention policies. + %% Let's wait for messages and hope these are less than the number of published ones + Ch1 = rabbit_ct_client_helpers:open_channel(Config, Server), + qos(Ch1, 100, false), + subscribe(Ch1, Q, false, 0), + + ?assert(length(receive_batch()) < 100). + +max_age(Config) -> + [Server | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + Q = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', Q, 0, 0}, + declare(Ch, Q, [{<<"x-queue-type">>, longstr, <<"stream">>}, + {<<"x-max-age">>, longstr, <<"10s">>}, + {<<"x-max-segment-size">>, long, 250}])), + + Payload = << <<"1">> || _ <- lists:seq(1, 500) >>, + + #'confirm.select_ok'{} = amqp_channel:call(Ch, #'confirm.select'{}), + amqp_channel:register_confirm_handler(Ch, self()), + [publish(Ch, Q, Payload) || _ <- lists:seq(1, 100)], + amqp_channel:wait_for_confirms(Ch, 5), + + timer:sleep(10000), + + %% Let's publish again so the new segments will trigger the retention policy + [publish(Ch, Q, Payload) || _ <- lists:seq(1, 100)], + amqp_channel:wait_for_confirms(Ch, 5), + + timer:sleep(5000), + + Ch1 = rabbit_ct_client_helpers:open_channel(Config, Server), + qos(Ch1, 200, false), + subscribe(Ch1, Q, false, 0), + ?assertEqual(100, length(receive_batch())). + +leader_failover(Config) -> + [Server1, Server2, Server3] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch1 = rabbit_ct_client_helpers:open_channel(Config, Server1), + Q = ?config(queue_name, Config), + + ?assertEqual({'queue.declare_ok', Q, 0, 0}, + declare(Ch1, Q, [{<<"x-queue-type">>, longstr, <<"stream">>}])), + + #'confirm.select_ok'{} = amqp_channel:call(Ch1, #'confirm.select'{}), + amqp_channel:register_confirm_handler(Ch1, self()), + [publish(Ch1, Q, <<"msg">>) || _ <- lists:seq(1, 100)], + amqp_channel:wait_for_confirms(Ch1, 5), + + check_leader_and_replicas(Config, Q, Server1, [Server2, Server3]), + + ok = rabbit_ct_broker_helpers:stop_node(Config, Server1), + timer:sleep(30000), + + [Info] = lists:filter( + fun(Props) -> + QName = rabbit_misc:r(<<"/">>, queue, Q), + lists:member({name, QName}, Props) + end, + rabbit_ct_broker_helpers:rpc(Config, 1, rabbit_amqqueue, + info_all, [<<"/">>, [name, leader, members]])), + NewLeader = proplists:get_value(leader, Info), + ?assert(NewLeader =/= Server1), + ok = rabbit_ct_broker_helpers:start_node(Config, Server1). + +initial_cluster_size_one(Config) -> + [Server1 | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server1), + Q = ?config(queue_name, Config), + + ?assertEqual({'queue.declare_ok', Q, 0, 0}, + declare(Ch, Q, [{<<"x-queue-type">>, longstr, <<"stream">>}, + {<<"x-initial-cluster-size">>, long, 1}])), + check_leader_and_replicas(Config, Q, Server1, []), + + ?assertMatch(#'queue.delete_ok'{}, + amqp_channel:call(Ch, #'queue.delete'{queue = Q})). + +initial_cluster_size_two(Config) -> + [Server1 | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server1), + Q = ?config(queue_name, Config), + + ?assertEqual({'queue.declare_ok', Q, 0, 0}, + declare(Ch, Q, [{<<"x-queue-type">>, longstr, <<"stream">>}, + {<<"x-initial-cluster-size">>, long, 2}])), + + [Info] = lists:filter( + fun(Props) -> + lists:member({name, rabbit_misc:r(<<"/">>, queue, Q)}, Props) + end, + rabbit_ct_broker_helpers:rpc(Config, 0, rabbit_amqqueue, + info_all, [<<"/">>, [name, leader, members]])), + ?assertEqual(Server1, proplists:get_value(leader, Info)), + ?assertEqual(1, length(proplists:get_value(members, Info))), + + ?assertMatch(#'queue.delete_ok'{}, + amqp_channel:call(Ch, #'queue.delete'{queue = Q})). + +initial_cluster_size_one_policy(Config) -> + [Server1 | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + ok = rabbit_ct_broker_helpers:set_policy( + Config, 0, <<"cluster-size">>, <<"initial_cluster_size_one_policy">>, <<"queues">>, + [{<<"initial-cluster-size">>, 1}]), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server1), + Q = ?config(queue_name, Config), + + ?assertEqual({'queue.declare_ok', Q, 0, 0}, + declare(Ch, Q, [{<<"x-queue-type">>, longstr, <<"stream">>}, + {<<"x-initial-cluster-size">>, long, 1}])), + check_leader_and_replicas(Config, Q, Server1, []), + + ?assertMatch(#'queue.delete_ok'{}, + amqp_channel:call(Ch, #'queue.delete'{queue = Q})), + + ok = rabbit_ct_broker_helpers:clear_policy(Config, 0, <<"cluster-size">>). + +leader_locator_client_local(Config) -> + [Server1, Server2, Server3] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server1), + Q = ?config(queue_name, Config), + + ?assertEqual({'queue.declare_ok', Q, 0, 0}, + declare(Ch, Q, [{<<"x-queue-type">>, longstr, <<"stream">>}, + {<<"x-queue-leader-locator">>, longstr, <<"client-local">>}])), + + [Info] = lists:filter( + fun(Props) -> + lists:member({name, rabbit_misc:r(<<"/">>, queue, Q)}, Props) + end, + rabbit_ct_broker_helpers:rpc(Config, 0, rabbit_amqqueue, + info_all, [<<"/">>, [name, leader]])), + ?assertEqual(Server1, proplists:get_value(leader, Info)), + + ?assertMatch(#'queue.delete_ok'{}, + amqp_channel:call(Ch, #'queue.delete'{queue = Q})), + + %% Try second node + Ch2 = rabbit_ct_client_helpers:open_channel(Config, Server2), + ?assertEqual({'queue.declare_ok', Q, 0, 0}, + declare(Ch2, Q, [{<<"x-queue-type">>, longstr, <<"stream">>}, + {<<"x-queue-leader-locator">>, longstr, <<"client-local">>}])), + + [Info2] = lists:filter( + fun(Props) -> + lists:member({name, rabbit_misc:r(<<"/">>, queue, Q)}, Props) + end, + rabbit_ct_broker_helpers:rpc(Config, 0, rabbit_amqqueue, + info_all, [<<"/">>, [name, leader]])), + ?assertEqual(Server2, proplists:get_value(leader, Info2)), + + ?assertMatch(#'queue.delete_ok'{}, + amqp_channel:call(Ch2, #'queue.delete'{queue = Q})), + + %% Try third node + Ch3 = rabbit_ct_client_helpers:open_channel(Config, Server3), + ?assertEqual({'queue.declare_ok', Q, 0, 0}, + declare(Ch3, Q, [{<<"x-queue-type">>, longstr, <<"stream">>}, + {<<"x-queue-leader-locator">>, longstr, <<"client-local">>}])), + + [Info3] = lists:filter( + fun(Props) -> + lists:member({name, rabbit_misc:r(<<"/">>, queue, Q)}, Props) + end, + rabbit_ct_broker_helpers:rpc(Config, 0, rabbit_amqqueue, + info_all, [<<"/">>, [name, leader]])), + ?assertEqual(Server3, proplists:get_value(leader, Info3)), + + ?assertMatch(#'queue.delete_ok'{}, + amqp_channel:call(Ch3, #'queue.delete'{queue = Q})). + +leader_locator_random(Config) -> + [Server1 | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server1), + Q = ?config(queue_name, Config), + + ?assertEqual({'queue.declare_ok', Q, 0, 0}, + declare(Ch, Q, [{<<"x-queue-type">>, longstr, <<"stream">>}, + {<<"x-queue-leader-locator">>, longstr, <<"random">>}])), + + [Info] = lists:filter( + fun(Props) -> + lists:member({name, rabbit_misc:r(<<"/">>, queue, Q)}, Props) + end, + rabbit_ct_broker_helpers:rpc(Config, 0, rabbit_amqqueue, + info_all, [<<"/">>, [name, leader]])), + Leader = proplists:get_value(leader, Info), + + ?assertMatch(#'queue.delete_ok'{}, + amqp_channel:call(Ch, #'queue.delete'{queue = Q})), + + repeat_until( + fun() -> + ?assertMatch(#'queue.delete_ok'{}, + amqp_channel:call(Ch, #'queue.delete'{queue = Q})), + + ?assertEqual({'queue.declare_ok', Q, 0, 0}, + declare(Ch, Q, [{<<"x-queue-type">>, longstr, <<"stream">>}, + {<<"x-queue-leader-locator">>, longstr, <<"random">>}])), + + [Info2] = lists:filter( + fun(Props) -> + lists:member({name, rabbit_misc:r(<<"/">>, queue, Q)}, Props) + end, + rabbit_ct_broker_helpers:rpc(Config, 0, rabbit_amqqueue, + info_all, [<<"/">>, [name, leader]])), + Leader2 = proplists:get_value(leader, Info2), + + Leader =/= Leader2 + end, 10). + +leader_locator_least_leaders(Config) -> + [Server1, Server2, Server3] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server1), + Q = ?config(queue_name, Config), + + Q1 = <<"q1">>, + Q2 = <<"q2">>, + ?assertEqual({'queue.declare_ok', Q1, 0, 0}, + declare(Ch, Q1, [{<<"x-queue-type">>, longstr, <<"stream">>}, + {<<"x-queue-leader-locator">>, longstr, <<"client-local">>}])), + ?assertEqual({'queue.declare_ok', Q2, 0, 0}, + declare(Ch, Q2, [{<<"x-queue-type">>, longstr, <<"stream">>}, + {<<"x-queue-leader-locator">>, longstr, <<"client-local">>}])), + + ?assertEqual({'queue.declare_ok', Q, 0, 0}, + declare(Ch, Q, [{<<"x-queue-type">>, longstr, <<"stream">>}, + {<<"x-queue-leader-locator">>, longstr, <<"least-leaders">>}])), + + [Info] = lists:filter( + fun(Props) -> + lists:member({name, rabbit_misc:r(<<"/">>, queue, Q)}, Props) + end, + rabbit_ct_broker_helpers:rpc(Config, 0, rabbit_amqqueue, + info_all, [<<"/">>, [name, leader]])), + Leader = proplists:get_value(leader, Info), + + ?assert(lists:member(Leader, [Server2, Server3])). + +leader_locator_policy(Config) -> + [Server | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + + ok = rabbit_ct_broker_helpers:set_policy( + Config, 0, <<"leader-locator">>, <<"leader_locator_.*">>, <<"queues">>, + [{<<"queue-leader-locator">>, <<"random">>}]), + + Q = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', Q, 0, 0}, + declare(Ch, Q, [{<<"x-queue-type">>, longstr, <<"stream">>}])), + + [Info] = rabbit_ct_broker_helpers:rpc(Config, 0, rabbit_amqqueue, + info_all, [<<"/">>, [policy, operator_policy, + effective_policy_definition, + name, leader]]), + + ?assertEqual(<<"leader-locator">>, proplists:get_value(policy, Info)), + ?assertEqual('', proplists:get_value(operator_policy, Info)), + ?assertEqual([{<<"queue-leader-locator">>, <<"random">>}], + proplists:get_value(effective_policy_definition, Info)), + + Leader = proplists:get_value(leader, Info), + + repeat_until( + fun() -> + ?assertMatch(#'queue.delete_ok'{}, + amqp_channel:call(Ch, #'queue.delete'{queue = Q})), + + ?assertEqual({'queue.declare_ok', Q, 0, 0}, + declare(Ch, Q, [{<<"x-queue-type">>, longstr, <<"stream">>}])), + + [Info2] = lists:filter( + fun(Props) -> + lists:member({name, rabbit_misc:r(<<"/">>, queue, Q)}, Props) + end, + rabbit_ct_broker_helpers:rpc(Config, 0, rabbit_amqqueue, + info_all, [<<"/">>, [name, leader]])), + Leader2 = proplists:get_value(leader, Info2), + Leader =/= Leader2 + end, 10), + + ok = rabbit_ct_broker_helpers:clear_policy(Config, 0, <<"leader-locator">>). + +repeat_until(_, 0) -> + ct:fail("Condition did not materialize in the expected amount of attempts"); +repeat_until(Fun, N) -> + case Fun() of + true -> ok; + false -> repeat_until(Fun, N - 1) + end. + +invalid_policy(Config) -> + [Server | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + Q = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', Q, 0, 0}, + declare(Ch, Q, [{<<"x-queue-type">>, longstr, <<"stream">>}])), + ok = rabbit_ct_broker_helpers:set_policy( + Config, 0, <<"ha">>, <<"invalid_policy.*">>, <<"queues">>, + [{<<"ha-mode">>, <<"all">>}]), + ok = rabbit_ct_broker_helpers:set_policy( + Config, 0, <<"ttl">>, <<"invalid_policy.*">>, <<"queues">>, + [{<<"message-ttl">>, 5}]), + + [Info] = rabbit_ct_broker_helpers:rpc(Config, 0, rabbit_amqqueue, + info_all, [<<"/">>, [policy, operator_policy, + effective_policy_definition]]), + + ?assertEqual('', proplists:get_value(policy, Info)), + ?assertEqual('', proplists:get_value(operator_policy, Info)), + ?assertEqual([], proplists:get_value(effective_policy_definition, Info)), + ok = rabbit_ct_broker_helpers:clear_policy(Config, 0, <<"ha">>), + ok = rabbit_ct_broker_helpers:clear_policy(Config, 0, <<"ttl">>). + +max_age_policy(Config) -> + [Server | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + Q = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', Q, 0, 0}, + declare(Ch, Q, [{<<"x-queue-type">>, longstr, <<"stream">>}])), + ok = rabbit_ct_broker_helpers:set_policy( + Config, 0, <<"age">>, <<"max_age_policy.*">>, <<"queues">>, + [{<<"max-age">>, <<"1Y">>}]), + + [Info] = rabbit_ct_broker_helpers:rpc(Config, 0, rabbit_amqqueue, + info_all, [<<"/">>, [policy, operator_policy, + effective_policy_definition]]), + + ?assertEqual(<<"age">>, proplists:get_value(policy, Info)), + ?assertEqual('', proplists:get_value(operator_policy, Info)), + ?assertEqual([{<<"max-age">>, <<"1Y">>}], + proplists:get_value(effective_policy_definition, Info)), + ok = rabbit_ct_broker_helpers:clear_policy(Config, 0, <<"age">>). + +max_segment_size_policy(Config) -> + [Server | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + Q = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', Q, 0, 0}, + declare(Ch, Q, [{<<"x-queue-type">>, longstr, <<"stream">>}])), + ok = rabbit_ct_broker_helpers:set_policy( + Config, 0, <<"segment">>, <<"max_segment_size.*">>, <<"queues">>, + [{<<"max-segment-size">>, 5000}]), + + [Info] = rabbit_ct_broker_helpers:rpc(Config, 0, rabbit_amqqueue, + info_all, [<<"/">>, [policy, operator_policy, + effective_policy_definition]]), + + ?assertEqual(<<"segment">>, proplists:get_value(policy, Info)), + ?assertEqual('', proplists:get_value(operator_policy, Info)), + ?assertEqual([{<<"max-segment-size">>, 5000}], + proplists:get_value(effective_policy_definition, Info)), + ok = rabbit_ct_broker_helpers:clear_policy(Config, 0, <<"segment">>). + +purge(Config) -> + Server = rabbit_ct_broker_helpers:get_node_config(Config, 0, nodename), + + Ch = rabbit_ct_client_helpers:open_channel(Config, Server), + Q = ?config(queue_name, Config), + ?assertEqual({'queue.declare_ok', Q, 0, 0}, + declare(Ch, Q, [{<<"x-queue-type">>, longstr, <<"stream">>}])), + + ?assertExit({{shutdown, {connection_closing, {server_initiated_close, 540, _}}}, _}, + amqp_channel:call(Ch, #'queue.purge'{queue = Q})). + +%%---------------------------------------------------------------------------- + +delete_queues() -> + [{ok, _} = rabbit_amqqueue:delete(Q, false, false, <<"dummy">>) + || Q <- rabbit_amqqueue:list()]. + +declare(Ch, Q) -> + declare(Ch, Q, []). + +declare(Ch, Q, Args) -> + amqp_channel:call(Ch, #'queue.declare'{queue = Q, + durable = true, + auto_delete = false, + arguments = Args}). +assert_queue_type(Server, Q, Expected) -> + Actual = get_queue_type(Server, Q), + Expected = Actual. + +get_queue_type(Server, Q0) -> + QNameRes = rabbit_misc:r(<<"/">>, queue, Q0), + {ok, Q1} = rpc:call(Server, rabbit_amqqueue, lookup, [QNameRes]), + amqqueue:get_type(Q1). + +check_leader_and_replicas(Config, Name, Leader, Replicas0) -> + QNameRes = rabbit_misc:r(<<"/">>, queue, Name), + [Info] = lists:filter( + fun(Props) -> + lists:member({name, QNameRes}, Props) + end, + rabbit_ct_broker_helpers:rpc(Config, 0, rabbit_amqqueue, + info_all, [<<"/">>, [name, leader, members]])), + ?assertEqual(Leader, proplists:get_value(leader, Info)), + Replicas = lists:sort(Replicas0), + ?assertEqual(Replicas, lists:sort(proplists:get_value(members, Info))). + +publish(Ch, Queue) -> + publish(Ch, Queue, <<"msg">>). + +publish(Ch, Queue, Msg) -> + ok = amqp_channel:cast(Ch, + #'basic.publish'{routing_key = Queue}, + #amqp_msg{props = #'P_basic'{delivery_mode = 2}, + payload = Msg}). + +subscribe(Ch, Queue, NoAck, Offset) -> + amqp_channel:subscribe(Ch, #'basic.consume'{queue = Queue, + no_ack = NoAck, + consumer_tag = <<"ctag">>, + arguments = [{<<"x-stream-offset">>, long, Offset}]}, + self()), + receive + #'basic.consume_ok'{consumer_tag = <<"ctag">>} -> + ok + end. + +qos(Ch, Prefetch, Global) -> + ?assertMatch(#'basic.qos_ok'{}, + amqp_channel:call(Ch, #'basic.qos'{global = Global, + prefetch_count = Prefetch})). + +receive_batch(Ch, N, N) -> + receive + {#'basic.deliver'{delivery_tag = DeliveryTag}, + #amqp_msg{props = #'P_basic'{headers = [{<<"x-stream-offset">>, long, N}]}}} -> + ok = amqp_channel:cast(Ch, #'basic.ack'{delivery_tag = DeliveryTag, + multiple = false}) + after 60000 -> + exit({missing_offset, N}) + end; +receive_batch(Ch, N, M) -> + receive + {_, + #amqp_msg{props = #'P_basic'{headers = [{<<"x-stream-offset">>, long, S}]}}} + when S < N -> + exit({unexpected_offset, S}); + {#'basic.deliver'{delivery_tag = DeliveryTag}, + #amqp_msg{props = #'P_basic'{headers = [{<<"x-stream-offset">>, long, N}]}}} -> + ok = amqp_channel:cast(Ch, #'basic.ack'{delivery_tag = DeliveryTag, + multiple = false}), + receive_batch(Ch, N + 1, M) + after 60000 -> + exit({missing_offset, N}) + end. + +receive_batch() -> + receive_batch([]). + +receive_batch(Acc) -> + receive + {#'basic.deliver'{delivery_tag = DeliveryTag}, _} -> + receive_batch([DeliveryTag | Acc]) + after 5000 -> + lists:reverse(Acc) + end. + +run_proper(Fun, Args, NumTests) -> + ?assertEqual( + true, + proper:counterexample( + erlang:apply(Fun, Args), + [{numtests, NumTests}, + {on_output, fun(".", _) -> ok; % don't print the '.'s on new lines + (F, A) -> ct:pal(?LOW_IMPORTANCE, F, A) + end}])). diff --git a/deps/rabbit/test/rabbitmq-env.bats b/deps/rabbit/test/rabbitmq-env.bats new file mode 100644 index 0000000000..4a016960c5 --- /dev/null +++ b/deps/rabbit/test/rabbitmq-env.bats @@ -0,0 +1,128 @@ +#!/usr/bin/env bats + +export RABBITMQ_SCRIPTS_DIR="$BATS_TEST_DIRNAME/../scripts" + +setup() { + export RABBITMQ_CONF_ENV_FILE="$BATS_TMPDIR/rabbitmq-env.$BATS_TEST_NAME.conf" +} + +@test "default Erlang scheduler bind type" { + source "$RABBITMQ_SCRIPTS_DIR/rabbitmq-env" + echo $RABBITMQ_SCHEDULER_BIND_TYPE + + echo "expected RABBITMQ_SERVER_ERL_ARGS to contain ' +stbt db ', but got: $RABBITMQ_SERVER_ERL_ARGS" + [[ $RABBITMQ_SERVER_ERL_ARGS == *" +stbt db "* ]] +} + +@test "can configure Erlang scheduler bind type via conf file" { + echo 'SCHEDULER_BIND_TYPE=u' > "$RABBITMQ_CONF_ENV_FILE" + source "$RABBITMQ_SCRIPTS_DIR/rabbitmq-env" + + echo "expected RABBITMQ_SERVER_ERL_ARGS to contain ' +stbt u ', but got: $RABBITMQ_SERVER_ERL_ARGS" + [[ $RABBITMQ_SERVER_ERL_ARGS == *" +stbt u "* ]] +} + +@test "can configure Erlang scheduler bind type via env" { + RABBITMQ_SCHEDULER_BIND_TYPE=tnnps source "$RABBITMQ_SCRIPTS_DIR/rabbitmq-env" + + echo "expected RABBITMQ_SERVER_ERL_ARGS to contain ' +stbt tnnps ', but got: $RABBITMQ_SERVER_ERL_ARGS" + [[ $RABBITMQ_SERVER_ERL_ARGS == *" +stbt tnnps "* ]] +} + +@test "Erlang scheduler bind type env takes precedence over conf file" { + echo 'SCHEDULER_BIND_TYPE=s' > "$RABBITMQ_CONF_ENV_FILE" + RABBITMQ_SCHEDULER_BIND_TYPE=nnps source "$RABBITMQ_SCRIPTS_DIR/rabbitmq-env" + + echo "expected RABBITMQ_SERVER_ERL_ARGS to contain ' +stbt nnps ', but got: $RABBITMQ_SERVER_ERL_ARGS" + [[ $RABBITMQ_SERVER_ERL_ARGS == *" +stbt nnps "* ]] +} + +@test "default Erlang distribution buffer size" { + source "$RABBITMQ_SCRIPTS_DIR/rabbitmq-env" + + echo "expected RABBITMQ_SERVER_ERL_ARGS to contain ' +zdbbl 128000 ', but got: $RABBITMQ_SERVER_ERL_ARGS" + [[ $RABBITMQ_SERVER_ERL_ARGS == *" +zdbbl 128000 "* ]] +} + +@test "can configure Erlang distribution buffer size via conf file" { + echo 'DISTRIBUTION_BUFFER_SIZE=123123' > "$RABBITMQ_CONF_ENV_FILE" + source "$RABBITMQ_SCRIPTS_DIR/rabbitmq-env" + + echo "expected RABBITMQ_SERVER_ERL_ARGS to contain ' +zdbbl 123123 ', but got: $RABBITMQ_SERVER_ERL_ARGS" + [[ $RABBITMQ_SERVER_ERL_ARGS == *" +zdbbl 123123 "* ]] +} + +@test "can configure Erlang distribution buffer size via env" { + RABBITMQ_DISTRIBUTION_BUFFER_SIZE=2000000 source "$RABBITMQ_SCRIPTS_DIR/rabbitmq-env" + + echo "expected RABBITMQ_SERVER_ERL_ARGS to contain ' +zdbbl 2000000 ', but got: $RABBITMQ_SERVER_ERL_ARGS" + [[ $RABBITMQ_SERVER_ERL_ARGS == *" +zdbbl 2000000 "* ]] +} + +@test "Erlang distribution buffer size env takes precedence over conf file" { + echo 'DISTRIBUTION_BUFFER_SIZE=3000000' > "$RABBITMQ_CONF_ENV_FILE" + RABBITMQ_DISTRIBUTION_BUFFER_SIZE=4000000 source "$RABBITMQ_SCRIPTS_DIR/rabbitmq-env" + + echo "expected RABBITMQ_SERVER_ERL_ARGS to contain ' +zdbbl 4000000 ', but got: $RABBITMQ_SERVER_ERL_ARGS" + [[ $RABBITMQ_SERVER_ERL_ARGS == *" +zdbbl 4000000 "* ]] +} + +@test "default Erlang maximum number of processes" { + source "$RABBITMQ_SCRIPTS_DIR/rabbitmq-env" + + echo "expected RABBITMQ_SERVER_ERL_ARGS to contain ' +P 1048576 ', but got: $RABBITMQ_SERVER_ERL_ARGS" + [[ $RABBITMQ_SERVER_ERL_ARGS == *" +P 1048576 "* ]] +} + +@test "can configure Erlang maximum number of processes via conf file" { + echo 'MAX_NUMBER_OF_PROCESSES=2000000' > "$RABBITMQ_CONF_ENV_FILE" + source "$RABBITMQ_SCRIPTS_DIR/rabbitmq-env" + + echo "expected RABBITMQ_SERVER_ERL_ARGS to contain ' +P 2000000 ', but got: $RABBITMQ_SERVER_ERL_ARGS" + [[ $RABBITMQ_SERVER_ERL_ARGS == *" +P 2000000 "* ]] +} + +@test "can configure Erlang maximum number of processes via env" { + RABBITMQ_MAX_NUMBER_OF_PROCESSES=3000000 source "$RABBITMQ_SCRIPTS_DIR/rabbitmq-env" + + echo "expected RABBITMQ_SERVER_ERL_ARGS to contain ' +P 3000000 ', but got: $RABBITMQ_SERVER_ERL_ARGS" + [[ $RABBITMQ_SERVER_ERL_ARGS == *" +P 3000000 "* ]] +} + +@test "Erlang maximum number of processes env takes precedence over conf file" { + echo 'MAX_NUMBER_OF_PROCESSES=4000000' > "$RABBITMQ_CONF_ENV_FILE" + RABBITMQ_MAX_NUMBER_OF_PROCESSES=5000000 source "$RABBITMQ_SCRIPTS_DIR/rabbitmq-env" + + echo "expected RABBITMQ_SERVER_ERL_ARGS to contain ' +P 5000000 ', but got: $RABBITMQ_SERVER_ERL_ARGS" + [[ $RABBITMQ_SERVER_ERL_ARGS == *" +P 5000000 "* ]] +} + +@test "default Erlang maximum number of atoms" { + source "$RABBITMQ_SCRIPTS_DIR/rabbitmq-env" + + echo "expected RABBITMQ_SERVER_ERL_ARGS to contain ' +t 5000000 ', but got: $RABBITMQ_SERVER_ERL_ARGS" + [[ $RABBITMQ_SERVER_ERL_ARGS == *" +t 5000000 "* ]] +} + +@test "can configure Erlang maximum number of atoms via conf file" { + echo 'MAX_NUMBER_OF_ATOMS=1000000' > "$RABBITMQ_CONF_ENV_FILE" + source "$RABBITMQ_SCRIPTS_DIR/rabbitmq-env" + + echo "expected RABBITMQ_SERVER_ERL_ARGS to contain ' +t 1000000 ', but got: $RABBITMQ_SERVER_ERL_ARGS" + [[ $RABBITMQ_SERVER_ERL_ARGS == *" +t 1000000 "* ]] +} + +@test "can configure Erlang maximum number of atoms via env" { + RABBITMQ_MAX_NUMBER_OF_ATOMS=2000000 source "$RABBITMQ_SCRIPTS_DIR/rabbitmq-env" + + echo "expected RABBITMQ_SERVER_ERL_ARGS to contain ' +t 2000000 ', but got: $RABBITMQ_SERVER_ERL_ARGS" + [[ $RABBITMQ_SERVER_ERL_ARGS == *" +t 2000000 "* ]] +} + +@test "Erlang maximum number of atoms env takes precedence over conf file" { + echo 'MAX_NUMBER_OF_ATOMS=3000000' > "$RABBITMQ_CONF_ENV_FILE" + RABBITMQ_MAX_NUMBER_OF_ATOMS=4000000 source "$RABBITMQ_SCRIPTS_DIR/rabbitmq-env" + + echo "expected RABBITMQ_SERVER_ERL_ARGS to contain ' +t 4000000 ', but got: $RABBITMQ_SERVER_ERL_ARGS" + [[ $RABBITMQ_SERVER_ERL_ARGS == *" +t 4000000 "* ]] +} diff --git a/deps/rabbit/test/rabbitmq_queues_cli_integration_SUITE.erl b/deps/rabbit/test/rabbitmq_queues_cli_integration_SUITE.erl new file mode 100644 index 0000000000..bf5e9ee79e --- /dev/null +++ b/deps/rabbit/test/rabbitmq_queues_cli_integration_SUITE.erl @@ -0,0 +1,139 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2017-2020 VMware, Inc. or its affiliates. All rights reserved. +%% +-module(rabbitmq_queues_cli_integration_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). + +-compile(export_all). + +all() -> + [ + {group, tests} + ]. + +groups() -> + [ + {tests, [], [ + shrink, + grow, + grow_invalid_node_filtered + ]} + ]. + +init_per_suite(Config) -> + case os:getenv("SECONDARY_UMBRELLA") of + false -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config); + _ -> + {skip, "growing and shrinking cannot be done in mixed mode"} + end. + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(tests, Config0) -> + NumNodes = 3, + Config1 = rabbit_ct_helpers:set_config( + Config0, [{rmq_nodes_count, NumNodes}, + {rmq_nodes_clustered, true}]), + Config2 = rabbit_ct_helpers:run_steps(Config1, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()), + case rabbit_ct_broker_helpers:enable_feature_flag(Config2, quorum_queue) of + ok -> + Config2; + Skip -> + end_per_group(tests, Config2), + Skip + end. + +end_per_group(tests, Config) -> + rabbit_ct_helpers:run_steps(Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()). + +init_per_testcase(Testcase, Config0) -> + rabbit_ct_helpers:ensure_rabbitmq_queues_cmd( + rabbit_ct_helpers:testcase_started(Config0, Testcase)). + +end_per_testcase(Testcase, Config0) -> + rabbit_ct_helpers:testcase_finished(Config0, Testcase). + +shrink(Config) -> + NodeConfig = rabbit_ct_broker_helpers:get_node_config(Config, 2), + Nodename2 = ?config(nodename, NodeConfig), + Ch = rabbit_ct_client_helpers:open_channel(Config, Nodename2), + %% declare a quorum queue + QName = "shrink1", + #'queue.declare_ok'{} = declare_qq(Ch, QName), + {ok, Out1} = rabbitmq_queues(Config, 0, ["shrink", Nodename2]), + ?assertMatch(#{{"/", "shrink1"} := {2, ok}}, parse_result(Out1)), + Nodename1 = rabbit_ct_broker_helpers:get_node_config(Config, 1, nodename), + {ok, Out2} = rabbitmq_queues(Config, 0, ["shrink", Nodename1]), + ?assertMatch(#{{"/", "shrink1"} := {1, ok}}, parse_result(Out2)), + Nodename0 = rabbit_ct_broker_helpers:get_node_config(Config, 0, nodename), + {ok, Out3} = rabbitmq_queues(Config, 0, ["shrink", Nodename0]), + ?assertMatch(#{{"/", "shrink1"} := {1, error}}, parse_result(Out3)), + ok. + +grow(Config) -> + NodeConfig = rabbit_ct_broker_helpers:get_node_config(Config, 2), + Nodename2 = ?config(nodename, NodeConfig), + Ch = rabbit_ct_client_helpers:open_channel(Config, Nodename2), + %% declare a quorum queue + QName = "grow1", + Args = [{<<"x-quorum-initial-group-size">>, long, 1}], + #'queue.declare_ok'{} = declare_qq(Ch, QName, Args), + Nodename0 = rabbit_ct_broker_helpers:get_node_config(Config, 0, nodename), + {ok, Out1} = rabbitmq_queues(Config, 0, ["grow", Nodename0, "all"]), + ?assertMatch(#{{"/", "grow1"} := {2, ok}}, parse_result(Out1)), + Nodename1 = rabbit_ct_broker_helpers:get_node_config(Config, 1, nodename), + {ok, Out2} = rabbitmq_queues(Config, 0, ["grow", Nodename1, "all"]), + ?assertMatch(#{{"/", "grow1"} := {3, ok}}, parse_result(Out2)), + + {ok, Out3} = rabbitmq_queues(Config, 0, ["grow", Nodename0, "all"]), + ?assertNotMatch(#{{"/", "grow1"} := _}, parse_result(Out3)), + ok. + +grow_invalid_node_filtered(Config) -> + NodeConfig = rabbit_ct_broker_helpers:get_node_config(Config, 2), + Nodename2 = ?config(nodename, NodeConfig), + Ch = rabbit_ct_client_helpers:open_channel(Config, Nodename2), + %% declare a quorum queue + QName = "grow-err", + Args = [{<<"x-quorum-initial-group-size">>, long, 1}], + #'queue.declare_ok'{} = declare_qq(Ch, QName, Args), + DummyNode = not_really_a_node@nothing, + {ok, Out1} = rabbitmq_queues(Config, 0, ["grow", DummyNode, "all"]), + ?assertNotMatch(#{{"/", "grow-err"} := _}, parse_result(Out1)), + ok. + +parse_result(S) -> + Lines = string:split(S, "\n", all), + maps:from_list( + [{{Vhost, QName}, + {erlang:list_to_integer(Size), case Result of + "ok" -> ok; + _ -> error + end}} + || [Vhost, QName, Size, Result] <- + [string:split(L, "\t", all) || L <- Lines]]). + +declare_qq(Ch, Q, Args0) -> + Args = [{<<"x-queue-type">>, longstr, <<"quorum">>}] ++ Args0, + amqp_channel:call(Ch, #'queue.declare'{queue = list_to_binary(Q), + durable = true, + auto_delete = false, + arguments = Args}). +declare_qq(Ch, Q) -> + declare_qq(Ch, Q, []). + +rabbitmq_queues(Config, N, Args) -> + rabbit_ct_broker_helpers:rabbitmq_queues(Config, N, ["--silent" | Args]). diff --git a/deps/rabbit/test/rabbitmqctl_integration_SUITE.erl b/deps/rabbit/test/rabbitmqctl_integration_SUITE.erl new file mode 100644 index 0000000000..9c689f5667 --- /dev/null +++ b/deps/rabbit/test/rabbitmqctl_integration_SUITE.erl @@ -0,0 +1,164 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2016-2020 VMware, Inc. or its affiliates. All rights reserved. +%% +-module(rabbitmqctl_integration_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-export([all/0 + ,groups/0 + ,init_per_suite/1 + ,end_per_suite/1 + ,init_per_group/2 + ,end_per_group/2 + ,init_per_testcase/2 + ,end_per_testcase/2 + ]). + +-export([list_queues_local/1 + ,list_queues_offline/1 + ,list_queues_online/1 + ,list_queues_stopped/1 + ]). + +all() -> + [ + {group, list_queues} + ]. + +groups() -> + [ + {list_queues, [], + [list_queues_local + ,list_queues_online + ,list_queues_offline + ,list_queues_stopped + ]} + ]. + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(list_queues, Config0) -> + NumNodes = 3, + Config = create_n_node_cluster(Config0, NumNodes), + Config1 = declare_some_queues(Config), + rabbit_ct_broker_helpers:stop_node(Config1, NumNodes - 1), + Config1; +init_per_group(_, Config) -> + Config. + +create_n_node_cluster(Config0, NumNodes) -> + Config1 = rabbit_ct_helpers:set_config( + Config0, [{rmq_nodes_count, NumNodes}, + {rmq_nodes_clustered, true}]), + rabbit_ct_helpers:run_steps(Config1, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()). + +declare_some_queues(Config) -> + Nodes = rabbit_ct_helpers:get_config(Config, rmq_nodes), + PerNodeQueues = [ declare_some_queues(Config, NodeNum) + || NodeNum <- lists:seq(0, length(Nodes)-1) ], + rabbit_ct_helpers:set_config(Config, {per_node_queues, PerNodeQueues}). + +declare_some_queues(Config, NodeNum) -> + {Conn, Chan} = rabbit_ct_client_helpers:open_connection_and_channel(Config, NodeNum), + NumQueues = 5, + Queues = [ list_to_binary(io_lib:format("queue-~b-on-node-~b", [QueueNum, NodeNum])) + || QueueNum <- lists:seq(1, NumQueues) ], + lists:foreach(fun (QueueName) -> + #'queue.declare_ok'{} = amqp_channel:call(Chan, #'queue.declare'{queue = QueueName, durable = true}) + end, Queues), + rabbit_ct_client_helpers:close_connection_and_channel(Conn, Chan), + Queues. + +end_per_group(list_queues, Config0) -> + Config1 = case rabbit_ct_helpers:get_config(Config0, save_config) of + undefined -> Config0; + C -> C + end, + rabbit_ct_helpers:run_steps(Config1, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()); +end_per_group(_, Config) -> + Config. + +init_per_testcase(list_queues_stopped, Config0) -> + %% Start node 3 to crash it's queues + rabbit_ct_broker_helpers:start_node(Config0, 2), + %% Make vhost "down" on nodes 2 and 3 + rabbit_ct_broker_helpers:force_vhost_failure(Config0, 1, <<"/">>), + rabbit_ct_broker_helpers:force_vhost_failure(Config0, 2, <<"/">>), + + rabbit_ct_broker_helpers:stop_node(Config0, 2), + rabbit_ct_helpers:testcase_started(Config0, list_queues_stopped); + +init_per_testcase(Testcase, Config0) -> + rabbit_ct_helpers:testcase_started(Config0, Testcase). + +end_per_testcase(Testcase, Config0) -> + rabbit_ct_helpers:testcase_finished(Config0, Testcase). + +%%---------------------------------------------------------------------------- +%% Test cases +%%---------------------------------------------------------------------------- +list_queues_local(Config) -> + Node1Queues = lists:sort(lists:nth(1, ?config(per_node_queues, Config))), + Node2Queues = lists:sort(lists:nth(2, ?config(per_node_queues, Config))), + assert_ctl_queues(Config, 0, ["--local"], Node1Queues), + assert_ctl_queues(Config, 1, ["--local"], Node2Queues), + ok. + +list_queues_online(Config) -> + Node1Queues = lists:sort(lists:nth(1, ?config(per_node_queues, Config))), + Node2Queues = lists:sort(lists:nth(2, ?config(per_node_queues, Config))), + OnlineQueues = Node1Queues ++ Node2Queues, + assert_ctl_queues(Config, 0, ["--online"], OnlineQueues), + assert_ctl_queues(Config, 1, ["--online"], OnlineQueues), + ok. + +list_queues_offline(Config) -> + Node3Queues = lists:sort(lists:nth(3, ?config(per_node_queues, Config))), + OfflineQueues = Node3Queues, + assert_ctl_queues(Config, 0, ["--offline"], OfflineQueues), + assert_ctl_queues(Config, 1, ["--offline"], OfflineQueues), + ok. + +list_queues_stopped(Config) -> + Node1Queues = lists:sort(lists:nth(1, ?config(per_node_queues, Config))), + Node2Queues = lists:sort(lists:nth(2, ?config(per_node_queues, Config))), + Node3Queues = lists:sort(lists:nth(3, ?config(per_node_queues, Config))), + + %% All queues are listed + ListedQueues = + [ {Name, State} + || [Name, State] <- rabbit_ct_broker_helpers:rabbitmqctl_list( + Config, 0, ["list_queues", "name", "state", "--no-table-headers"]) ], + + [ <<"running">> = proplists:get_value(Q, ListedQueues) || Q <- Node1Queues ], + %% Node is running. Vhost is down + [ <<"stopped">> = proplists:get_value(Q, ListedQueues) || Q <- Node2Queues ], + %% Node is not running. Vhost is down + [ <<"down">> = proplists:get_value(Q, ListedQueues) || Q <- Node3Queues ]. + +%%---------------------------------------------------------------------------- +%% Helpers +%%---------------------------------------------------------------------------- +assert_ctl_queues(Config, Node, Args, Expected0) -> + Expected = lists:sort(Expected0), + Got0 = run_list_queues(Config, Node, Args), + Got = lists:sort(lists:map(fun hd/1, Got0)), + ?assertMatch(Expected, Got). + +run_list_queues(Config, Node, Args) -> + rabbit_ct_broker_helpers:rabbitmqctl_list(Config, Node, ["list_queues"] ++ Args ++ ["name", "--no-table-headers"]). diff --git a/deps/rabbit/test/rabbitmqctl_shutdown_SUITE.erl b/deps/rabbit/test/rabbitmqctl_shutdown_SUITE.erl new file mode 100644 index 0000000000..6365f91d47 --- /dev/null +++ b/deps/rabbit/test/rabbitmqctl_shutdown_SUITE.erl @@ -0,0 +1,110 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2017-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(rabbitmqctl_shutdown_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). + +-compile(export_all). + +all() -> + [ + {group, running_node}, + {group, non_running_node} + ]. + +groups() -> + [ + {running_node, [], [ + successful_shutdown + ]}, + {non_running_node, [], [ + nothing_to_shutdown + ]} + ]. + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(running_node, Config) -> + rabbit_ct_helpers:set_config(Config, [ + {rmq_nodename_suffix, ?MODULE}, + {need_start, true} + ]); +init_per_group(non_running_node, Config) -> + rabbit_ct_helpers:set_config(Config, [ + {rmq_nodename_suffix, ?MODULE} + ]). + +end_per_group(running_node, Config) -> + rabbit_ct_helpers:run_teardown_steps(Config, []); +end_per_group(non_running_node, Config) -> + rabbit_ct_helpers:run_teardown_steps(Config, []). + +init_per_testcase(Testcase, Config0) -> + Config1 = case ?config(need_start, Config0) of + true -> + rabbit_ct_helpers:run_setup_steps(Config0, + rabbit_ct_broker_helpers:setup_steps() ++ + [fun save_node/1]); + _ -> + rabbit_ct_helpers:set_config(Config0, + [{node, non_existent_node@localhost}]) + end, + rabbit_ct_helpers:testcase_started(Config1, Testcase). + +end_per_testcase(Testcase, Config0) -> + Config1 = case ?config(need_start, Config0) of + true -> + rabbit_ct_helpers:run_teardown_steps(Config0, + rabbit_ct_broker_helpers:teardown_steps()); + _ -> Config0 + end, + rabbit_ct_helpers:testcase_finished(Config1, Testcase). + +save_node(Config) -> + Node = rabbit_ct_broker_helpers:get_node_config(Config, 0, nodename), + rabbit_ct_helpers:set_config(Config, [{node, Node}]). + +successful_shutdown(Config) -> + Node = ?config(node, Config), + Pid = node_pid(Node), + ok = shutdown_ok(Node), + false = erlang_pid_is_running(Pid), + false = node_is_running(Node). + + +nothing_to_shutdown(Config) -> + Node = ?config(node, Config), + + { error, 69, _ } = + rabbit_ct_broker_helpers:control_action(shutdown, Node, []). + +node_pid(Node) -> + Val = rpc:call(Node, os, getpid, []), + true = is_list(Val), + list_to_integer(Val). + +erlang_pid_is_running(Pid) -> + rabbit_misc:is_os_process_alive(integer_to_list(Pid)). + +node_is_running(Node) -> + net_adm:ping(Node) == pong. + +shutdown_ok(Node) -> + %% Start a command + {stream, Stream} = rabbit_ct_broker_helpers:control_action(shutdown, Node, []), + %% Execute command steps. Each step will output a binary string + Lines = 'Elixir.Enum':to_list(Stream), + ct:pal("Command output ~p ~n", [Lines]), + [true = is_binary(Line) || Line <- Lines], + ok. diff --git a/deps/rabbit/test/signal_handling_SUITE.erl b/deps/rabbit/test/signal_handling_SUITE.erl new file mode 100644 index 0000000000..551f456039 --- /dev/null +++ b/deps/rabbit/test/signal_handling_SUITE.erl @@ -0,0 +1,160 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(signal_handling_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-export([suite/0, + all/0, + groups/0, + init_per_suite/1, + end_per_suite/1, + init_per_group/2, + end_per_group/2, + init_per_testcase/2, + end_per_testcase/2, + + send_sighup/1, + send_sigterm/1, + send_sigtstp/1 + ]). + +suite() -> + [{timetrap, {minutes, 5}}]. + +all() -> + [ + {group, signal_sent_to_pid_in_pidfile}, + {group, signal_sent_to_pid_from_os_getpid} + ]. + +groups() -> + Signals = [sighup, + sigterm, + sigtstp], + Tests = [list_to_existing_atom(rabbit_misc:format("send_~s", [Signal])) + || Signal <- Signals], + [ + {signal_sent_to_pid_in_pidfile, [], Tests}, + {signal_sent_to_pid_from_os_getpid, [], Tests} + ]. + +%% ------------------------------------------------------------------- +%% Testsuite setup/teardown. +%% ------------------------------------------------------------------- + +init_per_suite(Config) -> + case os:type() of + {unix, _} -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config); + _ -> + {skip, "This testsuite is only relevant on Unix"} + end. + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(_, Config) -> + Config. + +end_per_group(_, Config) -> + Config. + +init_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_started(Config, Testcase), + ClusterSize = 1, + TestNumber = rabbit_ct_helpers:testcase_number(Config, ?MODULE, Testcase), + Config1 = rabbit_ct_helpers:set_config( + Config, + [ + {rmq_nodename_suffix, Testcase}, + {tcp_ports_base, {skip_n_nodes, TestNumber * ClusterSize}} + ]), + rabbit_ct_helpers:run_steps(Config1, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()). + +end_per_testcase(Testcase, Config) -> + Config1 = rabbit_ct_helpers:run_steps(Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()), + rabbit_ct_helpers:testcase_finished(Config1, Testcase). + +%% ------------------------------------------------------------------- +%% Testcases. +%% ------------------------------------------------------------------- + +send_sighup(Config) -> + {PidFile, Pid} = get_pidfile_and_pid(Config), + + %% A SIGHUP signal should be ignored and the node should still be + %% running. + send_signal(Pid, "HUP"), + timer:sleep(10000), + A = rabbit_ct_broker_helpers:get_node_config(Config, 0, nodename), + ?assert(rabbit_ct_broker_helpers:rpc(Config, A, rabbit, is_running, [])), + ?assert(filelib:is_regular(PidFile)). + +send_sigterm(Config) -> + {PidFile, Pid} = get_pidfile_and_pid(Config), + + %% After sending a SIGTERM to the process, we expect the node to + %% exit. + send_signal(Pid, "TERM"), + wait_for_node_exit(Pid), + + %% After a clean exit, the PID file should be removed. + ?assertNot(filelib:is_regular(PidFile)). + +send_sigtstp(Config) -> + {PidFile, Pid} = get_pidfile_and_pid(Config), + + %% A SIGHUP signal should be ignored and the node should still be + %% running. + send_signal(Pid, "TSTP"), + timer:sleep(10000), + A = rabbit_ct_broker_helpers:get_node_config(Config, 0, nodename), + ?assert(rabbit_ct_broker_helpers:rpc(Config, A, rabbit, is_running, [])), + ?assert(filelib:is_regular(PidFile)). + +get_pidfile_and_pid(Config) -> + PidFile = rabbit_ct_broker_helpers:get_node_config(Config, 0, pid_file), + ?assert(filelib:is_regular(PidFile)), + + %% We send the signal to either the process referenced in the + %% PID file or the Erlang VM process. Up-to 3.8.x, they can be + %% the different process because the PID file may reference the + %% rabbitmq-server(8) script wrapper. + [{name, Group} | _] = ?config(tc_group_properties, Config), + Pid = case Group of + signal_sent_to_pid_in_pidfile -> + {ok, P} = file:read_file(PidFile), + string:trim(P, trailing, [$\r,$\n]); + signal_sent_to_pid_from_os_getpid -> + A = rabbit_ct_broker_helpers:get_node_config( + Config, 0, nodename), + rabbit_ct_broker_helpers:rpc(Config, A, os, getpid, []) + end, + {PidFile, Pid}. + +send_signal(Pid, Signal) -> + Cmd = ["kill", + "-" ++ Signal, + Pid], + ?assertMatch({ok, _}, rabbit_ct_helpers:exec(Cmd)). + +wait_for_node_exit(Pid) -> + case rabbit_misc:is_os_process_alive(Pid) of + true -> + timer:sleep(1000), + wait_for_node_exit(Pid); + false -> + ok + end. diff --git a/deps/rabbit/test/simple_ha_SUITE.erl b/deps/rabbit/test/simple_ha_SUITE.erl new file mode 100644 index 0000000000..8b2c1d6ebb --- /dev/null +++ b/deps/rabbit/test/simple_ha_SUITE.erl @@ -0,0 +1,371 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(simple_ha_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-compile(export_all). + +-define(DELAY, 8000). + +all() -> + [ + {group, cluster_size_2}, + {group, cluster_size_3} + ]. + +groups() -> + RejectTests = [ + rejects_survive_stop, + rejects_survive_policy + ], + [ + {cluster_size_2, [], [ + rapid_redeclare, + declare_synchrony, + clean_up_exclusive_queues, + clean_up_and_redeclare_exclusive_queues_on_other_nodes + ]}, + {cluster_size_3, [], [ + consume_survives_stop, + consume_survives_policy, + auto_resume, + auto_resume_no_ccn_client, + confirms_survive_stop, + confirms_survive_policy, + {overflow_reject_publish, [], RejectTests}, + {overflow_reject_publish_dlx, [], RejectTests} + ]} + ]. + +%% ------------------------------------------------------------------- +%% Testsuite setup/teardown. +%% ------------------------------------------------------------------- + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(cluster_size_2, Config) -> + rabbit_ct_helpers:set_config(Config, [ + {rmq_nodes_count, 2} + ]); +init_per_group(cluster_size_3, Config) -> + rabbit_ct_helpers:set_config(Config, [ + {rmq_nodes_count, 3} + ]); +init_per_group(overflow_reject_publish, Config) -> + rabbit_ct_helpers:set_config(Config, [ + {overflow, <<"reject-publish">>} + ]); +init_per_group(overflow_reject_publish_dlx, Config) -> + rabbit_ct_helpers:set_config(Config, [ + {overflow, <<"reject-publish-dlx">>} + ]). + +end_per_group(_, Config) -> + Config. + +init_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_started(Config, Testcase), + ClusterSize = ?config(rmq_nodes_count, Config), + TestNumber = rabbit_ct_helpers:testcase_number(Config, ?MODULE, Testcase), + Config1 = rabbit_ct_helpers:set_config(Config, [ + {rmq_nodes_clustered, true}, + {rmq_nodename_suffix, Testcase}, + {tcp_ports_base, {skip_n_nodes, TestNumber * ClusterSize}} + ]), + rabbit_ct_helpers:run_steps(Config1, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps() ++ [ + fun rabbit_ct_broker_helpers:set_ha_policy_all/1 + ]). + +end_per_testcase(Testcase, Config) -> + Config1 = rabbit_ct_helpers:run_steps(Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()), + rabbit_ct_helpers:testcase_finished(Config1, Testcase). + +%% ------------------------------------------------------------------- +%% Testcases. +%% ------------------------------------------------------------------- + +rapid_redeclare(Config) -> + A = rabbit_ct_broker_helpers:get_node_config(Config, 0, nodename), + Ch = rabbit_ct_client_helpers:open_channel(Config, A), + Queue = <<"test">>, + [begin + amqp_channel:call(Ch, #'queue.declare'{queue = Queue, + durable = true}), + amqp_channel:call(Ch, #'queue.delete'{queue = Queue}) + end || _I <- lists:seq(1, 20)], + ok. + +%% Check that by the time we get a declare-ok back, the mirrors are up +%% and in Mnesia. +declare_synchrony(Config) -> + [Rabbit, Hare] = rabbit_ct_broker_helpers:get_node_configs(Config, + nodename), + RabbitCh = rabbit_ct_client_helpers:open_channel(Config, Rabbit), + HareCh = rabbit_ct_client_helpers:open_channel(Config, Hare), + Q = <<"mirrored-queue">>, + declare(RabbitCh, Q), + amqp_channel:call(RabbitCh, #'confirm.select'{}), + amqp_channel:cast(RabbitCh, #'basic.publish'{routing_key = Q}, + #amqp_msg{props = #'P_basic'{delivery_mode = 2}}), + amqp_channel:wait_for_confirms(RabbitCh), + rabbit_ct_broker_helpers:kill_node(Config, Rabbit), + + #'queue.declare_ok'{message_count = 1} = declare(HareCh, Q), + ok. + +declare(Ch, Name) -> + amqp_channel:call(Ch, #'queue.declare'{durable = true, queue = Name}). + +%% Ensure that exclusive queues are cleaned up when part of ha cluster +%% and node is killed abruptly then restarted +clean_up_exclusive_queues(Config) -> + QName = <<"excl">>, + rabbit_ct_broker_helpers:set_ha_policy(Config, 0, <<".*">>, <<"all">>), + [A, B] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + ChA = rabbit_ct_client_helpers:open_channel(Config, A), + amqp_channel:call(ChA, #'queue.declare'{queue = QName, + exclusive = true}), + ok = rabbit_ct_broker_helpers:kill_node(Config, A), + timer:sleep(?DELAY), + [] = rabbit_ct_broker_helpers:rpc(Config, B, rabbit_amqqueue, list, []), + ok = rabbit_ct_broker_helpers:start_node(Config, A), + timer:sleep(?DELAY), + [[],[]] = rabbit_ct_broker_helpers:rpc_all(Config, rabbit_amqqueue, list, []), + ok. + +clean_up_and_redeclare_exclusive_queues_on_other_nodes(Config) -> + QueueCount = 10, + QueueNames = lists:map(fun(N) -> + NBin = erlang:integer_to_binary(N), + <<"exclusive-q-", NBin/binary>> + end, lists:seq(1, QueueCount)), + [A, B] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + Conn = rabbit_ct_client_helpers:open_unmanaged_connection(Config, A), + {ok, Ch} = amqp_connection:open_channel(Conn), + + LocationMinMasters = [ + {<<"x-queue-master-locator">>, longstr, <<"min-masters">>} + ], + lists:foreach(fun(QueueName) -> + declare_exclusive(Ch, QueueName, LocationMinMasters), + subscribe(Ch, QueueName) + end, QueueNames), + + ok = rabbit_ct_broker_helpers:kill_node(Config, B), + + Cancels = receive_cancels([]), + ?assert(length(Cancels) > 0), + + RemaniningQueues = rabbit_ct_broker_helpers:rpc(Config, A, rabbit_amqqueue, list, []), + + ?assertEqual(length(RemaniningQueues), QueueCount - length(Cancels)), + + lists:foreach(fun(QueueName) -> + declare_exclusive(Ch, QueueName, LocationMinMasters), + true = rabbit_ct_client_helpers:publish(Ch, QueueName, 1), + subscribe(Ch, QueueName) + end, QueueNames), + Messages = receive_messages([]), + ?assertEqual(10, length(Messages)), + ok = rabbit_ct_client_helpers:close_connection(Conn). + + +consume_survives_stop(Cf) -> consume_survives(Cf, fun stop/2, true). +consume_survives_sigkill(Cf) -> consume_survives(Cf, fun sigkill/2, true). +consume_survives_policy(Cf) -> consume_survives(Cf, fun policy/2, true). +auto_resume(Cf) -> consume_survives(Cf, fun sigkill/2, false). +auto_resume_no_ccn_client(Cf) -> consume_survives(Cf, fun sigkill/2, false, + false). + +confirms_survive_stop(Cf) -> confirms_survive(Cf, fun stop/2). +confirms_survive_policy(Cf) -> confirms_survive(Cf, fun policy/2). + +rejects_survive_stop(Cf) -> rejects_survive(Cf, fun stop/2). +rejects_survive_policy(Cf) -> rejects_survive(Cf, fun policy/2). + +%%---------------------------------------------------------------------------- + +consume_survives(Config, DeathFun, CancelOnFailover) -> + consume_survives(Config, DeathFun, CancelOnFailover, true). + +consume_survives(Config, + DeathFun, CancelOnFailover, CCNSupported) -> + [A, B, C] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + Msgs = rabbit_ct_helpers:cover_work_factor(Config, 20000), + Channel1 = rabbit_ct_client_helpers:open_channel(Config, A), + Channel2 = rabbit_ct_client_helpers:open_channel(Config, B), + Channel3 = rabbit_ct_client_helpers:open_channel(Config, C), + + %% declare the queue on the master, mirrored to the two mirrors + Queue = <<"test">>, + amqp_channel:call(Channel1, #'queue.declare'{queue = Queue, + auto_delete = false}), + + %% start up a consumer + ConsCh = case CCNSupported of + true -> Channel2; + false -> Port = rabbit_ct_broker_helpers:get_node_config( + Config, B, tcp_port_amqp), + open_incapable_channel(Port) + end, + ConsumerPid = rabbit_ha_test_consumer:create( + ConsCh, Queue, self(), CancelOnFailover, Msgs), + + %% send a bunch of messages from the producer + ProducerPid = rabbit_ha_test_producer:create(Channel3, Queue, + self(), false, Msgs), + DeathFun(Config, A), + %% verify that the consumer got all msgs, or die - the await_response + %% calls throw an exception if anything goes wrong.... + ct:pal("awaiting produce ~w", [ProducerPid]), + rabbit_ha_test_producer:await_response(ProducerPid), + ct:pal("awaiting consumer ~w", [ConsumerPid]), + rabbit_ha_test_consumer:await_response(ConsumerPid), + ok. + +confirms_survive(Config, DeathFun) -> + [A, B, _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + Msgs = rabbit_ct_helpers:cover_work_factor(Config, 20000), + Node1Channel = rabbit_ct_client_helpers:open_channel(Config, A), + Node2Channel = rabbit_ct_client_helpers:open_channel(Config, B), + + %% declare the queue on the master, mirrored to the two mirrors + Queue = <<"test">>, + amqp_channel:call(Node1Channel,#'queue.declare'{queue = Queue, + auto_delete = false, + durable = true}), + + %% send one message to ensure the channel is flowing + amqp_channel:register_confirm_handler(Node1Channel, self()), + #'confirm.select_ok'{} = amqp_channel:call(Node1Channel, #'confirm.select'{}), + + Payload = <<"initial message">>, + ok = amqp_channel:call(Node1Channel, + #'basic.publish'{routing_key = Queue}, + #amqp_msg{payload = Payload}), + + ok = receive + #'basic.ack'{multiple = false} -> ok; + #'basic.nack'{multiple = false} -> message_nacked + after + 5000 -> confirm_not_received + end, + + %% send a bunch of messages from the producer + ProducerPid = rabbit_ha_test_producer:create(Node2Channel, Queue, + self(), true, Msgs), + DeathFun(Config, A), + rabbit_ha_test_producer:await_response(ProducerPid), + ok. + +rejects_survive(Config, DeathFun) -> + [A, B, _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + Msgs = rabbit_ct_helpers:cover_work_factor(Config, 20000), + Node1Channel = rabbit_ct_client_helpers:open_channel(Config, A), + Node2Channel = rabbit_ct_client_helpers:open_channel(Config, B), + + %% declare the queue on the master, mirrored to the two mirrors + XOverflow = ?config(overflow, Config), + Queue = <<"test_rejects", "_", XOverflow/binary>>, + amqp_channel:call(Node1Channel,#'queue.declare'{queue = Queue, + auto_delete = false, + durable = true, + arguments = [{<<"x-max-length">>, long, 1}, + {<<"x-overflow">>, longstr, XOverflow}]}), + + amqp_channel:register_confirm_handler(Node1Channel, self()), + #'confirm.select_ok'{} = amqp_channel:call(Node1Channel, #'confirm.select'{}), + + Payload = <<"there can be only one">>, + ok = amqp_channel:call(Node1Channel, + #'basic.publish'{routing_key = Queue}, + #amqp_msg{payload = Payload}), + + ok = receive + #'basic.ack'{multiple = false} -> ok; + #'basic.nack'{multiple = false} -> message_nacked + after + 5000 -> confirm_not_received + end, + + %% send a bunch of messages from the producer. They should all be nacked, as the queue is full. + ProducerPid = rabbit_ha_test_producer:create(Node2Channel, Queue, + self(), true, Msgs, nacks), + DeathFun(Config, A), + rabbit_ha_test_producer:await_response(ProducerPid), + + {#'basic.get_ok'{}, #amqp_msg{payload = Payload}} = + amqp_channel:call(Node2Channel, #'basic.get'{queue = Queue}), + %% There is only one message. + #'basic.get_empty'{} = amqp_channel:call(Node2Channel, #'basic.get'{queue = Queue}), + ok. + + + +stop(Config, Node) -> + rabbit_ct_broker_helpers:stop_node_after(Config, Node, 50). + +sigkill(Config, Node) -> + rabbit_ct_broker_helpers:kill_node_after(Config, Node, 50). + +policy(Config, Node)-> + Nodes = [ + rabbit_misc:atom_to_binary(N) + || N <- rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + N =/= Node], + rabbit_ct_broker_helpers:set_ha_policy(Config, Node, <<".*">>, + {<<"nodes">>, Nodes}). + +open_incapable_channel(NodePort) -> + Props = [{<<"capabilities">>, table, []}], + {ok, ConsConn} = + amqp_connection:start(#amqp_params_network{port = NodePort, + client_properties = Props}), + {ok, Ch} = amqp_connection:open_channel(ConsConn), + Ch. + +declare_exclusive(Ch, QueueName, Args) -> + Declare = #'queue.declare'{queue = QueueName, + exclusive = true, + arguments = Args + }, + #'queue.declare_ok'{} = amqp_channel:call(Ch, Declare). + +subscribe(Ch, QueueName) -> + ConsumeOk = amqp_channel:call(Ch, #'basic.consume'{queue = QueueName, + no_ack = true}), + #'basic.consume_ok'{} = ConsumeOk, + receive ConsumeOk -> ok after ?DELAY -> throw(consume_ok_timeout) end. + +receive_cancels(Cancels) -> + receive + #'basic.cancel'{} = C -> + receive_cancels([C|Cancels]) + after ?DELAY -> + Cancels + end. + +receive_messages(All) -> + receive + {#'basic.deliver'{}, Msg} -> + receive_messages([Msg|All]) + after ?DELAY -> + lists:reverse(All) + end. diff --git a/deps/rabbit/test/single_active_consumer_SUITE.erl b/deps/rabbit/test/single_active_consumer_SUITE.erl new file mode 100644 index 0000000000..59f2b6e83d --- /dev/null +++ b/deps/rabbit/test/single_active_consumer_SUITE.erl @@ -0,0 +1,376 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2018-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(single_active_consumer_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). + +-compile(export_all). + +-define(TIMEOUT, 30000). + +all() -> + [ + {group, classic_queue}, {group, quorum_queue} + ]. + +groups() -> + [ + {classic_queue, [], [ + all_messages_go_to_one_consumer, + fallback_to_another_consumer_when_first_one_is_cancelled, + fallback_to_another_consumer_when_exclusive_consumer_channel_is_cancelled, + fallback_to_another_consumer_when_first_one_is_cancelled_manual_acks, + amqp_exclusive_consume_fails_on_exclusive_consumer_queue + ]}, + {quorum_queue, [], [ + all_messages_go_to_one_consumer, + fallback_to_another_consumer_when_first_one_is_cancelled, + fallback_to_another_consumer_when_exclusive_consumer_channel_is_cancelled, + fallback_to_another_consumer_when_first_one_is_cancelled_manual_acks, + basic_get_is_unsupported + %% amqp_exclusive_consume_fails_on_exclusive_consumer_queue % Exclusive consume not implemented in QQ + ]} + ]. + +init_per_suite(Config0) -> + rabbit_ct_helpers:log_environment(), + Config1 = rabbit_ct_helpers:set_config(Config0, [ + {rmq_nodename_suffix, ?MODULE} + ]), + Config = rabbit_ct_helpers:merge_app_env( + Config1, {rabbit, [{quorum_tick_interval, 1000}]}), + rabbit_ct_helpers:run_setup_steps(Config, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()). + +init_per_group(classic_queue, Config) -> + [{single_active_consumer_queue_declare, + #'queue.declare'{arguments = [ + {<<"x-single-active-consumer">>, bool, true}, + {<<"x-queue-type">>, longstr, <<"classic">>} + ], + auto_delete = true} + } | Config]; +init_per_group(quorum_queue, Config) -> + Ret = rabbit_ct_broker_helpers:rpc( + Config, 0, rabbit_feature_flags, enable, [quorum_queue]), + case Ret of + ok -> + [{single_active_consumer_queue_declare, + #'queue.declare'{ + arguments = [ + {<<"x-single-active-consumer">>, bool, true}, + {<<"x-queue-type">>, longstr, <<"quorum">>} + ], + durable = true, exclusive = false, auto_delete = false} + } | Config]; + Error -> + {skip, {"Quorum queues are unsupported", Error}} + end. + +end_per_group(_, Config) -> + Config. + +init_per_testcase(Testcase, Config0) -> + Config = [{queue_name, atom_to_binary(Testcase, utf8)} | Config0], + rabbit_ct_helpers:testcase_started(Config, Testcase). +end_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_finished(Config, Testcase). + +all_messages_go_to_one_consumer(Config) -> + {C, Ch} = connection_and_channel(Config), + Q = queue_declare(Ch, Config), + MessageCount = 5, + ConsumerPid = spawn(?MODULE, consume, [{self(), {maps:new(), 0}, MessageCount}]), + #'basic.consume_ok'{consumer_tag = CTag1} = + amqp_channel:subscribe(Ch, #'basic.consume'{queue = Q, no_ack = true}, ConsumerPid), + #'basic.consume_ok'{consumer_tag = CTag2} = + amqp_channel:subscribe(Ch, #'basic.consume'{queue = Q, no_ack = true}, ConsumerPid), + + Publish = #'basic.publish'{exchange = <<>>, routing_key = Q}, + [amqp_channel:cast(Ch, Publish, #amqp_msg{payload = <<"foobar">>}) || _X <- lists:seq(1, MessageCount)], + + receive + {consumer_done, {MessagesPerConsumer, MessageCount}} -> + ?assertEqual(MessageCount, MessageCount), + ?assertEqual(2, maps:size(MessagesPerConsumer)), + ?assertEqual(MessageCount, maps:get(CTag1, MessagesPerConsumer)), + ?assertEqual(0, maps:get(CTag2, MessagesPerConsumer)) + after ?TIMEOUT -> + flush(), + throw(failed) + end, + + amqp_connection:close(C), + ok. + +fallback_to_another_consumer_when_first_one_is_cancelled(Config) -> + {C, Ch} = connection_and_channel(Config), + Q = queue_declare(Ch, Config), + MessageCount = 10, + ConsumerPid = spawn(?MODULE, consume, [{self(), {maps:new(), 0}, MessageCount}]), + #'basic.consume_ok'{consumer_tag = CTag1} = + amqp_channel:subscribe(Ch, #'basic.consume'{queue = Q, no_ack = true}, ConsumerPid), + #'basic.consume_ok'{consumer_tag = CTag2} = + amqp_channel:subscribe(Ch, #'basic.consume'{queue = Q, no_ack = true}, ConsumerPid), + #'basic.consume_ok'{consumer_tag = CTag3} = + amqp_channel:subscribe(Ch, #'basic.consume'{queue = Q, no_ack = true}, ConsumerPid), + + Publish = #'basic.publish'{exchange = <<>>, routing_key = Q}, + [amqp_channel:cast(Ch, Publish, #amqp_msg{payload = <<"foobar">>}) || _X <- lists:seq(1, MessageCount div 2)], + + {ok, {MessagesPerConsumer1, _}} = wait_for_messages(MessageCount div 2), + FirstActiveConsumerInList = maps:keys(maps:filter(fun(_CTag, Count) -> Count > 0 end, MessagesPerConsumer1)), + ?assertEqual(1, length(FirstActiveConsumerInList)), + + FirstActiveConsumer = lists:nth(1, FirstActiveConsumerInList), + #'basic.cancel_ok'{} = amqp_channel:call(Ch, #'basic.cancel'{consumer_tag = FirstActiveConsumer}), + + {cancel_ok, FirstActiveConsumer} = wait_for_cancel_ok(), + + [amqp_channel:cast(Ch, Publish, #amqp_msg{payload = <<"foobar">>}) || _X <- lists:seq(MessageCount div 2 + 1, MessageCount - 1)], + + {ok, {MessagesPerConsumer2, _}} = wait_for_messages(MessageCount div 2 - 1), + SecondActiveConsumerInList = maps:keys(maps:filter( + fun(CTag, Count) -> Count > 0 andalso CTag /= FirstActiveConsumer end, + MessagesPerConsumer2) + ), + ?assertEqual(1, length(SecondActiveConsumerInList)), + SecondActiveConsumer = lists:nth(1, SecondActiveConsumerInList), + + #'basic.cancel_ok'{} = amqp_channel:call(Ch, #'basic.cancel'{consumer_tag = SecondActiveConsumer}), + + amqp_channel:cast(Ch, Publish, #amqp_msg{payload = <<"foobar">>}), + ?assertMatch({ok, _}, wait_for_messages(1)), + + LastActiveConsumer = lists:nth(1, lists:delete(FirstActiveConsumer, lists:delete(SecondActiveConsumer, [CTag1, CTag2, CTag3]))), + + receive + {consumer_done, {MessagesPerConsumer, MessageCount}} -> + ?assertEqual(MessageCount, MessageCount), + ?assertEqual(3, maps:size(MessagesPerConsumer)), + ?assertEqual(MessageCount div 2, maps:get(FirstActiveConsumer, MessagesPerConsumer)), + ?assertEqual(MessageCount div 2 - 1, maps:get(SecondActiveConsumer, MessagesPerConsumer)), + ?assertEqual(1, maps:get(LastActiveConsumer, MessagesPerConsumer)) + after ?TIMEOUT -> + flush(), + throw(failed) + end, + + amqp_connection:close(C), + ok. + +fallback_to_another_consumer_when_first_one_is_cancelled_manual_acks(Config) -> + %% Let's ensure that although the consumer is cancelled we still keep the unacked + %% messages and accept acknowledgments on them. + [Server | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + {C, Ch} = connection_and_channel(Config), + Q = queue_declare(Ch, Config), + #'basic.consume_ok'{} = + amqp_channel:subscribe(Ch, #'basic.consume'{queue = Q, no_ack = false}, self()), + #'basic.consume_ok'{} = + amqp_channel:subscribe(Ch, #'basic.consume'{queue = Q, no_ack = false}, self()), + #'basic.consume_ok'{} = + amqp_channel:subscribe(Ch, #'basic.consume'{queue = Q, no_ack = false}, self()), + Consumers0 = rpc:call(Server, rabbit_amqqueue, consumers_all, [<<"/">>]), + ?assertMatch([_, _, _], lists:filter(fun(Props) -> + Resource = proplists:get_value(queue_name, Props), + Q == Resource#resource.name + end, Consumers0)), + + Publish = #'basic.publish'{exchange = <<>>, routing_key = Q}, + [amqp_channel:cast(Ch, Publish, #amqp_msg{payload = P}) || P <- [<<"msg1">>, <<"msg2">>]], + + {CTag, DTag1} = receive_deliver(), + {_CTag, DTag2} = receive_deliver(), + + quorum_queue_utils:wait_for_messages(Config, [[Q, <<"2">>, <<"0">>, <<"2">>]]), + #'basic.cancel_ok'{} = amqp_channel:call(Ch, #'basic.cancel'{consumer_tag = CTag}), + + receive + #'basic.cancel_ok'{consumer_tag = CTag} -> + ok + end, + Consumers1 = rpc:call(Server, rabbit_amqqueue, consumers_all, [<<"/">>]), + ?assertMatch([_, _], lists:filter(fun(Props) -> + Resource = proplists:get_value(queue_name, Props), + Q == Resource#resource.name + end, Consumers1)), + quorum_queue_utils:wait_for_messages(Config, [[Q, <<"2">>, <<"0">>, <<"2">>]]), + + [amqp_channel:cast(Ch, Publish, #amqp_msg{payload = P}) || P <- [<<"msg3">>, <<"msg4">>]], + + quorum_queue_utils:wait_for_messages(Config, [[Q, <<"4">>, <<"0">>, <<"4">>]]), + amqp_channel:cast(Ch, #'basic.ack'{delivery_tag = DTag1}), + amqp_channel:cast(Ch, #'basic.ack'{delivery_tag = DTag2}), + quorum_queue_utils:wait_for_messages(Config, [[Q, <<"2">>, <<"0">>, <<"2">>]]), + + amqp_connection:close(C), + ok. + +fallback_to_another_consumer_when_exclusive_consumer_channel_is_cancelled(Config) -> + {C, Ch} = connection_and_channel(Config), + {C1, Ch1} = connection_and_channel(Config), + {C2, Ch2} = connection_and_channel(Config), + {C3, Ch3} = connection_and_channel(Config), + Q = queue_declare(Ch, Config), + MessageCount = 10, + Consumer1Pid = spawn(?MODULE, consume, [{self(), {maps:new(), 0}, MessageCount div 2}]), + Consumer2Pid = spawn(?MODULE, consume, [{self(), {maps:new(), 0}, MessageCount div 2 - 1}]), + Consumer3Pid = spawn(?MODULE, consume, [{self(), {maps:new(), 0}, MessageCount div 2 - 1}]), + #'basic.consume_ok'{consumer_tag = CTag1} = + amqp_channel:subscribe(Ch1, #'basic.consume'{queue = Q, no_ack = true, consumer_tag = <<"1">>}, Consumer1Pid), + #'basic.consume_ok'{} = + amqp_channel:subscribe(Ch2, #'basic.consume'{queue = Q, no_ack = true, consumer_tag = <<"2">>}, Consumer2Pid), + #'basic.consume_ok'{} = + amqp_channel:subscribe(Ch3, #'basic.consume'{queue = Q, no_ack = true, consumer_tag = <<"3">>}, Consumer3Pid), + + Publish = #'basic.publish'{exchange = <<>>, routing_key = Q}, + [amqp_channel:cast(Ch, Publish, #amqp_msg{payload = <<"foobar">>}) || _X <- lists:seq(1, MessageCount div 2)], + + {MessagesPerConsumer1, MessageCount1} = consume_results(), + ?assertEqual(MessageCount div 2, MessageCount1), + ?assertEqual(1, maps:size(MessagesPerConsumer1)), + ?assertEqual(MessageCount div 2, maps:get(CTag1, MessagesPerConsumer1)), + + ok = amqp_channel:close(Ch1), + + [amqp_channel:cast(Ch, Publish, #amqp_msg{payload = <<"foobar">>}) || _X <- lists:seq(MessageCount div 2 + 1, MessageCount - 1)], + + {MessagesPerConsumer2, MessageCount2} = consume_results(), + ?assertEqual(MessageCount div 2 - 1, MessageCount2), + ?assertEqual(1, maps:size(MessagesPerConsumer2)), + + ok = amqp_channel:close(Ch2), + + amqp_channel:cast(Ch, Publish, #amqp_msg{payload = <<"poison">>}), + + {MessagesPerConsumer3, MessageCount3} = consume_results(), + ?assertEqual(1, MessageCount3), + ?assertEqual(1, maps:size(MessagesPerConsumer3)), + + [amqp_connection:close(Conn) || Conn <- [C1, C2, C3, C]], + ok. + +basic_get_is_unsupported(Config) -> + {C, Ch} = connection_and_channel(Config), + Q = queue_declare(Ch, Config), + + ?assertExit( + {{shutdown, {server_initiated_close, 405, _}}, _}, + amqp_channel:call(Ch, #'basic.get'{queue = Q, no_ack = false})), + + amqp_connection:close(C), + ok. + +amqp_exclusive_consume_fails_on_exclusive_consumer_queue(Config) -> + {C, Ch} = connection_and_channel(Config), + Q = queue_declare(Ch, Config), + ?assertExit( + {{shutdown, {server_initiated_close, 403, _}}, _}, + amqp_channel:call(Ch, #'basic.consume'{queue = Q, exclusive = true}) + ), + amqp_connection:close(C), + ok. + +connection_and_channel(Config) -> + C = rabbit_ct_client_helpers:open_unmanaged_connection(Config), + {ok, Ch} = amqp_connection:open_channel(C), + {C, Ch}. + +queue_declare(Channel, Config) -> + QueueName = ?config(queue_name, Config), + Declare0 = ?config(single_active_consumer_queue_declare, Config), + Declare = Declare0#'queue.declare'{queue = QueueName}, + #'queue.declare_ok'{queue = Q} = amqp_channel:call(Channel, Declare), + Q. + +consume({Parent, State, 0}) -> + Parent ! {consumer_done, State}; +consume({Parent, {MessagesPerConsumer, MessageCount}, CountDown}) -> + receive + #'basic.consume_ok'{consumer_tag = CTag} -> + consume({Parent, {maps:put(CTag, 0, MessagesPerConsumer), MessageCount}, CountDown}); + {#'basic.deliver'{consumer_tag = CTag}, #amqp_msg{payload = <<"poison">>}} -> + Parent ! {consumer_done, + {maps:update_with(CTag, fun(V) -> V + 1 end, MessagesPerConsumer), + MessageCount + 1}}; + {#'basic.deliver'{consumer_tag = CTag}, _Content} -> + NewState = {maps:update_with(CTag, fun(V) -> V + 1 end, MessagesPerConsumer), + MessageCount + 1}, + Parent ! {message, NewState}, + consume({Parent, NewState, CountDown - 1}); + #'basic.cancel_ok'{consumer_tag = CTag} -> + Parent ! {cancel_ok, CTag}, + consume({Parent, {MessagesPerConsumer, MessageCount}, CountDown}); + _ -> + consume({Parent, {MessagesPerConsumer, MessageCount}, CountDown}) + after ?TIMEOUT -> + Parent ! {consumer_timeout, {MessagesPerConsumer, MessageCount}}, + flush(), + exit(consumer_timeout) + end. + +consume_results() -> + receive + {consumer_done, {MessagesPerConsumer, MessageCount}} -> + {MessagesPerConsumer, MessageCount}; + {consumer_timeout, {MessagesPerConsumer, MessageCount}} -> + {MessagesPerConsumer, MessageCount}; + _ -> + consume_results() + after ?TIMEOUT -> + flush(), + throw(failed) + end. + +wait_for_messages(ExpectedCount) -> + wait_for_messages(ExpectedCount, {}). + +wait_for_messages(0, State) -> + {ok, State}; +wait_for_messages(ExpectedCount, State) -> + receive + {message, {MessagesPerConsumer, MessageCount}} -> + wait_for_messages(ExpectedCount - 1, {MessagesPerConsumer, MessageCount}) + after 5000 -> + {missing, ExpectedCount, State} + end. + +wait_for_cancel_ok() -> + receive + {cancel_ok, CTag} -> + {cancel_ok, CTag} + after 5000 -> + throw(consumer_cancel_ok_timeout) + end. + +receive_deliver() -> + receive + {#'basic.deliver'{consumer_tag = CTag, + delivery_tag = DTag}, _} -> + {CTag, DTag} + after 5000 -> + exit(deliver_timeout) + end. + +flush() -> + receive + Msg -> + ct:pal("flushed: ~w~n", [Msg]), + flush() + after 10 -> + ok + end. diff --git a/deps/rabbit/test/sync_detection_SUITE.erl b/deps/rabbit/test/sync_detection_SUITE.erl new file mode 100644 index 0000000000..55a86b7b3d --- /dev/null +++ b/deps/rabbit/test/sync_detection_SUITE.erl @@ -0,0 +1,243 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(sync_detection_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). + +-compile(export_all). + +-define(LOOP_RECURSION_DELAY, 100). + +all() -> + [ + {group, cluster_size_2}, + {group, cluster_size_3} + ]. + +groups() -> + [ + {cluster_size_2, [], [ + follower_synchronization + ]}, + {cluster_size_3, [], [ + follower_synchronization_ttl + ]} + ]. + +%% ------------------------------------------------------------------- +%% Testsuite setup/teardown. +%% ------------------------------------------------------------------- + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(cluster_size_2, Config) -> + rabbit_ct_helpers:set_config(Config, [{rmq_nodes_count, 2}]); +init_per_group(cluster_size_3, Config) -> + rabbit_ct_helpers:set_config(Config, [{rmq_nodes_count, 3}]). + +end_per_group(_, Config) -> + Config. + +init_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_started(Config, Testcase), + ClusterSize = ?config(rmq_nodes_count, Config), + TestNumber = rabbit_ct_helpers:testcase_number(Config, ?MODULE, Testcase), + Config1 = rabbit_ct_helpers:set_config(Config, [ + {rmq_nodes_count, ClusterSize}, + {rmq_nodes_clustered, true}, + {rmq_nodename_suffix, Testcase}, + {tcp_ports_base, {skip_n_nodes, TestNumber * ClusterSize}} + ]), + rabbit_ct_helpers:run_steps(Config1, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps() ++ [ + fun rabbit_ct_broker_helpers:set_ha_policy_two_pos/1, + fun rabbit_ct_broker_helpers:set_ha_policy_two_pos_batch_sync/1 + ]). + +end_per_testcase(Testcase, Config) -> + Config1 = rabbit_ct_helpers:run_steps(Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()), + rabbit_ct_helpers:testcase_finished(Config1, Testcase). + +%% ------------------------------------------------------------------- +%% Testcases. +%% ------------------------------------------------------------------- + +follower_synchronization(Config) -> + [Master, Slave] = rabbit_ct_broker_helpers:get_node_configs(Config, + nodename), + Channel = rabbit_ct_client_helpers:open_channel(Config, Master), + Queue = <<"ha.two.test">>, + #'queue.declare_ok'{} = + amqp_channel:call(Channel, #'queue.declare'{queue = Queue, + auto_delete = false}), + + %% The comments on the right are the queue length and the pending acks on + %% the master. + rabbit_ct_broker_helpers:stop_broker(Config, Slave), + + %% We get and ack one message when the mirror is down, and check that when we + %% start the mirror it's not marked as synced until ack the message. We also + %% publish another message when the mirror is up. + send_dummy_message(Channel, Queue), % 1 - 0 + {#'basic.get_ok'{delivery_tag = Tag1}, _} = + amqp_channel:call(Channel, #'basic.get'{queue = Queue}), % 0 - 1 + + rabbit_ct_broker_helpers:start_broker(Config, Slave), + + follower_unsynced(Master, Queue), + send_dummy_message(Channel, Queue), % 1 - 1 + follower_unsynced(Master, Queue), + + amqp_channel:cast(Channel, #'basic.ack'{delivery_tag = Tag1}), % 1 - 0 + + follower_synced(Master, Queue), + + %% We restart the mirror and we send a message, so that the mirror will only + %% have one of the messages. + rabbit_ct_broker_helpers:stop_broker(Config, Slave), + rabbit_ct_broker_helpers:start_broker(Config, Slave), + + send_dummy_message(Channel, Queue), % 2 - 0 + + follower_unsynced(Master, Queue), + + %% We reject the message that the mirror doesn't have, and verify that it's + %% still unsynced + {#'basic.get_ok'{delivery_tag = Tag2}, _} = + amqp_channel:call(Channel, #'basic.get'{queue = Queue}), % 1 - 1 + follower_unsynced(Master, Queue), + amqp_channel:cast(Channel, #'basic.reject'{ delivery_tag = Tag2, + requeue = true }), % 2 - 0 + follower_unsynced(Master, Queue), + {#'basic.get_ok'{delivery_tag = Tag3}, _} = + amqp_channel:call(Channel, #'basic.get'{queue = Queue}), % 1 - 1 + amqp_channel:cast(Channel, #'basic.ack'{delivery_tag = Tag3}), % 1 - 0 + follower_synced(Master, Queue), + {#'basic.get_ok'{delivery_tag = Tag4}, _} = + amqp_channel:call(Channel, #'basic.get'{queue = Queue}), % 0 - 1 + amqp_channel:cast(Channel, #'basic.ack'{delivery_tag = Tag4}), % 0 - 0 + follower_synced(Master, Queue). + +follower_synchronization_ttl(Config) -> + [Master, Slave, DLX] = rabbit_ct_broker_helpers:get_node_configs(Config, + nodename), + Channel = rabbit_ct_client_helpers:open_channel(Config, Master), + DLXChannel = rabbit_ct_client_helpers:open_channel(Config, DLX), + + %% We declare a DLX queue to wait for messages to be TTL'ed + DLXQueue = <<"dlx-queue">>, + #'queue.declare_ok'{} = + amqp_channel:call(Channel, #'queue.declare'{queue = DLXQueue, + auto_delete = false}), + + TestMsgTTL = 5000, + Queue = <<"ha.two.test">>, + %% Sadly we need fairly high numbers for the TTL because starting/stopping + %% nodes takes a fair amount of time. + Args = [{<<"x-message-ttl">>, long, TestMsgTTL}, + {<<"x-dead-letter-exchange">>, longstr, <<>>}, + {<<"x-dead-letter-routing-key">>, longstr, DLXQueue}], + #'queue.declare_ok'{} = + amqp_channel:call(Channel, #'queue.declare'{queue = Queue, + auto_delete = false, + arguments = Args}), + + follower_synced(Master, Queue), + + %% All unknown + rabbit_ct_broker_helpers:stop_broker(Config, Slave), + send_dummy_message(Channel, Queue), + send_dummy_message(Channel, Queue), + rabbit_ct_broker_helpers:start_broker(Config, Slave), + follower_unsynced(Master, Queue), + wait_for_messages(DLXQueue, DLXChannel, 2), + follower_synced(Master, Queue), + + %% 1 unknown, 1 known + rabbit_ct_broker_helpers:stop_broker(Config, Slave), + send_dummy_message(Channel, Queue), + rabbit_ct_broker_helpers:start_broker(Config, Slave), + follower_unsynced(Master, Queue), + send_dummy_message(Channel, Queue), + follower_unsynced(Master, Queue), + wait_for_messages(DLXQueue, DLXChannel, 2), + follower_synced(Master, Queue), + + %% %% both known + send_dummy_message(Channel, Queue), + send_dummy_message(Channel, Queue), + follower_synced(Master, Queue), + wait_for_messages(DLXQueue, DLXChannel, 2), + follower_synced(Master, Queue), + + ok. + +send_dummy_message(Channel, Queue) -> + Payload = <<"foo">>, + Publish = #'basic.publish'{exchange = <<>>, routing_key = Queue}, + amqp_channel:cast(Channel, Publish, #amqp_msg{payload = Payload}). + +follower_pids(Node, Queue) -> + {ok, Q} = rpc:call(Node, rabbit_amqqueue, lookup, + [rabbit_misc:r(<<"/">>, queue, Queue)]), + SSP = synchronised_slave_pids, + [{SSP, Pids}] = rpc:call(Node, rabbit_amqqueue, info, [Q, [SSP]]), + case Pids of + '' -> []; + _ -> Pids + end. + +%% The mnesia synchronization takes a while, but we don't want to wait for the +%% test to fail, since the timetrap is quite high. +wait_for_sync_status(Status, Node, Queue) -> + Max = 90000 / ?LOOP_RECURSION_DELAY, + wait_for_sync_status(0, Max, Status, Node, Queue). + +wait_for_sync_status(N, Max, Status, Node, Queue) when N >= Max -> + erlang:error({sync_status_max_tries_failed, + [{queue, Queue}, + {node, Node}, + {expected_status, Status}, + {max_tried, Max}]}); +wait_for_sync_status(N, Max, Status, Node, Queue) -> + Synced = length(follower_pids(Node, Queue)) =:= 1, + case Synced =:= Status of + true -> ok; + false -> timer:sleep(?LOOP_RECURSION_DELAY), + wait_for_sync_status(N + 1, Max, Status, Node, Queue) + end. + +follower_synced(Node, Queue) -> + wait_for_sync_status(true, Node, Queue). + +follower_unsynced(Node, Queue) -> + wait_for_sync_status(false, Node, Queue). + +wait_for_messages(Queue, Channel, N) -> + Sub = #'basic.consume'{queue = Queue}, + #'basic.consume_ok'{consumer_tag = CTag} = amqp_channel:call(Channel, Sub), + receive + #'basic.consume_ok'{} -> ok + end, + lists:foreach( + fun (_) -> receive + {#'basic.deliver'{delivery_tag = Tag}, _Content} -> + amqp_channel:cast(Channel, + #'basic.ack'{delivery_tag = Tag}) + end + end, lists:seq(1, N)), + amqp_channel:call(Channel, #'basic.cancel'{consumer_tag = CTag}). diff --git a/deps/rabbit/test/temp/head_message_timestamp_tests.py b/deps/rabbit/test/temp/head_message_timestamp_tests.py new file mode 100755 index 0000000000..6698b88b7b --- /dev/null +++ b/deps/rabbit/test/temp/head_message_timestamp_tests.py @@ -0,0 +1,131 @@ +#!/usr/bin/python +# +# Tests for the SLA patch which adds the head_message_timestamp queue stat. +# Uses both the management interface via rabbitmqadmin and the AMQP interface via Pika. +# There's no particular reason to have used rabbitmqadmin other than saving some bulk. +# Similarly, the separate declaration of exchanges and queues is just a preference +# following a typical enterprise policy where admin users create these resources. + +from datetime import datetime +import json +import pika +import os +import sys +from time import clock, mktime, sleep +import unittest + +# Uses the rabbitmqadmin script. +# To be imported this must be given a .py suffix and placed on the Python path +from rabbitmqadmin import * + +TEXCH = 'head-message-timestamp-test' +TQUEUE = 'head-message-timestamp-test-queue' + +TIMEOUT_SECS = 10 + +TIMESTAMP1 = mktime(datetime(2010,1,1,12,00,01).timetuple()) +TIMESTAMP2 = mktime(datetime(2010,1,1,12,00,02).timetuple()) + +AMQP_PORT = 99 + +DELIVERY_MODE = 2 +DURABLE = False + +def log(msg): + print("\nINFO: " + msg) + +class RabbitTestCase(unittest.TestCase): + def setUp(self): + parser.set_conflict_handler('resolve') + (options, args) = make_configuration() + AMQP_PORT = int(options.port) - 10000 + + self.mgmt = Management(options, args) + self.mgmt.put('/exchanges/%2f/' + TEXCH, '{"type" : "fanout", "durable":' + str(DURABLE).lower() + '}') + self.mgmt.put('/queues/%2f/' + TQUEUE, '{"auto_delete":false,"durable":' + str(DURABLE).lower() + ',"arguments":[]}') + self.mgmt.post('/bindings/%2f/e/' + TEXCH + '/q/' + TQUEUE, '{"routing_key": ".*", "arguments":[]}') + self.credentials = pika.PlainCredentials(options.username, options.password) + parameters = pika.ConnectionParameters(options.hostname, port=AMQP_PORT, credentials=self.credentials) + self.connection = pika.BlockingConnection(parameters) + self.channel = self.connection.channel() + + def tearDown(self): + parser.set_conflict_handler('resolve') + (options, args) = make_configuration() + self.mgmt = Management(options, args) + self.mgmt.delete('/queues/%2f/' + TQUEUE) + self.mgmt.delete('/exchanges/%2f/' + TEXCH) + +class RabbitSlaTestCase(RabbitTestCase): + def get_queue_stats(self, queue_name): + stats_str = self.mgmt.get('/queues/%2f/' + queue_name) + return json.loads(stats_str) + + def get_head_message_timestamp(self, queue_name): + return self.get_queue_stats(queue_name)["head_message_timestamp"] + + def send(self, message, timestamp=None): + self.channel.basic_publish(TEXCH, '', message, + pika.BasicProperties(content_type='text/plain', + delivery_mode=DELIVERY_MODE, + timestamp=timestamp)) + log("Sent message with body: " + str(message)) + + def receive(self, queue): + method_frame, header_frame, body = self.channel.basic_get(queue = queue) + log("Received message with body: " + str(body)) + return method_frame.delivery_tag, body + + def ack(self, delivery_tag): + self.channel.basic_ack(delivery_tag) + + def nack(self, delivery_tag): + self.channel.basic_nack(delivery_tag) + + def wait_for_new_timestamp(self, queue, old_timestamp): + stats_wait_start = clock() + while ((clock() - stats_wait_start) < TIMEOUT_SECS and + self.get_head_message_timestamp(queue) == old_timestamp): + sleep(0.1) + log('Queue stats updated in ' + str(clock() - stats_wait_start) + ' secs.') + return self.get_head_message_timestamp(queue) + + # TESTS + + def test_no_timestamp_when_queue_is_empty(self): + assert self.get_head_message_timestamp(TQUEUE) == '' + + def test_has_timestamp_when_first_msg_is_added(self): + self.send('Msg1', TIMESTAMP1) + stats_timestamp = self.wait_for_new_timestamp(TQUEUE, '') + assert stats_timestamp == TIMESTAMP1 + + def test_no_timestamp_when_last_msg_is_removed(self): + self.send('Msg1', TIMESTAMP1) + stats_timestamp = self.wait_for_new_timestamp(TQUEUE, '') + tag, body = self.receive(TQUEUE) + self.ack(tag) + stats_timestamp = self.wait_for_new_timestamp(TQUEUE, TIMESTAMP1) + assert stats_timestamp == '' + + def test_timestamp_updated_when_msg_is_removed(self): + self.send('Msg1', TIMESTAMP1) + stats_timestamp = self.wait_for_new_timestamp(TQUEUE, '') + self.send('Msg2', TIMESTAMP2) + tag, body = self.receive(TQUEUE) + self.ack(tag) + stats_timestamp = self.wait_for_new_timestamp(TQUEUE, TIMESTAMP1) + assert stats_timestamp == TIMESTAMP2 + + def test_timestamp_not_updated_before_msg_is_acked(self): + self.send('Msg1', TIMESTAMP1) + stats_timestamp = self.wait_for_new_timestamp(TQUEUE, '') + tag, body = self.receive(TQUEUE) + sleep(1) # Allow time for update to appear if it was going to (it shouldn't) + assert self.get_head_message_timestamp(TQUEUE) == TIMESTAMP1 + self.ack(tag) + +if __name__ == '__main__': + unittest.main(verbosity = 2) + + diff --git a/deps/rabbit/test/temp/rabbitmqadmin.py b/deps/rabbit/test/temp/rabbitmqadmin.py new file mode 100755 index 0000000000..cdddd56497 --- /dev/null +++ b/deps/rabbit/test/temp/rabbitmqadmin.py @@ -0,0 +1,934 @@ +#!/usr/bin/env python + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. +# Copyright (c) 2010-2020 VMware, Inc. or its affiliates. All rights reserved + +import sys +if sys.version_info[0] < 2 or sys.version_info[1] < 6: + print "Sorry, rabbitmqadmin requires at least Python 2.6." + sys.exit(1) + +from ConfigParser import ConfigParser, NoSectionError +from optparse import OptionParser, TitledHelpFormatter +import httplib +import urllib +import urlparse +import base64 +import json +import os +import socket + +VERSION = '0.0.0' + +LISTABLE = {'connections': {'vhost': False}, + 'channels': {'vhost': False}, + 'consumers': {'vhost': True}, + 'exchanges': {'vhost': True}, + 'queues': {'vhost': True}, + 'bindings': {'vhost': True}, + 'users': {'vhost': False}, + 'vhosts': {'vhost': False}, + 'permissions': {'vhost': False}, + 'nodes': {'vhost': False}, + 'parameters': {'vhost': False, + 'json': ['value']}, + 'policies': {'vhost': False, + 'json': ['definition']}} + +SHOWABLE = {'overview': {'vhost': False}} + +PROMOTE_COLUMNS = ['vhost', 'name', 'type', + 'source', 'destination', 'destination_type', 'routing_key'] + +URIS = { + 'exchange': '/exchanges/{vhost}/{name}', + 'queue': '/queues/{vhost}/{name}', + 'binding': '/bindings/{vhost}/e/{source}/{destination_char}/{destination}', + 'binding_del':'/bindings/{vhost}/e/{source}/{destination_char}/{destination}/{properties_key}', + 'vhost': '/vhosts/{name}', + 'user': '/users/{name}', + 'permission': '/permissions/{vhost}/{user}', + 'parameter': '/parameters/{component}/{vhost}/{name}', + 'policy': '/policies/{vhost}/{name}' + } + +DECLARABLE = { + 'exchange': {'mandatory': ['name', 'type'], + 'json': ['arguments'], + 'optional': {'auto_delete': 'false', 'durable': 'true', + 'internal': 'false', 'arguments': {}}}, + 'queue': {'mandatory': ['name'], + 'json': ['arguments'], + 'optional': {'auto_delete': 'false', 'durable': 'true', + 'arguments': {}, 'node': None}}, + 'binding': {'mandatory': ['source', 'destination'], + 'json': ['arguments'], + 'optional': {'destination_type': 'queue', + 'routing_key': '', 'arguments': {}}}, + 'vhost': {'mandatory': ['name'], + 'optional': {'tracing': None}}, + 'user': {'mandatory': ['name', 'password', 'tags'], + 'optional': {}}, + 'permission': {'mandatory': ['vhost', 'user', 'configure', 'write', 'read'], + 'optional': {}}, + 'parameter': {'mandatory': ['component', 'name', 'value'], + 'json': ['value'], + 'optional': {}}, + # Priority is 'json' to convert to int + 'policy': {'mandatory': ['name', 'pattern', 'definition'], + 'json': ['definition', 'priority'], + 'optional': {'priority' : 0, 'apply-to': None}} + } + +DELETABLE = { + 'exchange': {'mandatory': ['name']}, + 'queue': {'mandatory': ['name']}, + 'binding': {'mandatory': ['source', 'destination_type', 'destination', + 'properties_key']}, + 'vhost': {'mandatory': ['name']}, + 'user': {'mandatory': ['name']}, + 'permission': {'mandatory': ['vhost', 'user']}, + 'parameter': {'mandatory': ['component', 'name']}, + 'policy': {'mandatory': ['name']} + } + +CLOSABLE = { + 'connection': {'mandatory': ['name'], + 'optional': {}, + 'uri': '/connections/{name}'} + } + +PURGABLE = { + 'queue': {'mandatory': ['name'], + 'optional': {}, + 'uri': '/queues/{vhost}/{name}/contents'} + } + +EXTRA_VERBS = { + 'publish': {'mandatory': ['routing_key'], + 'optional': {'payload': None, + 'exchange': 'amq.default', + 'payload_encoding': 'string'}, + 'uri': '/exchanges/{vhost}/{exchange}/publish'}, + 'get': {'mandatory': ['queue'], + 'optional': {'count': '1', 'requeue': 'true', + 'payload_file': None, 'encoding': 'auto'}, + 'uri': '/queues/{vhost}/{queue}/get'} +} + +for k in DECLARABLE: + DECLARABLE[k]['uri'] = URIS[k] + +for k in DELETABLE: + DELETABLE[k]['uri'] = URIS[k] + DELETABLE[k]['optional'] = {} +DELETABLE['binding']['uri'] = URIS['binding_del'] + +def short_usage(): + return "rabbitmqadmin [options] subcommand" + +def title(name): + return "\n%s\n%s\n\n" % (name, '=' * len(name)) + +def subcommands_usage(): + usage = """Usage +===== + """ + short_usage() + """ + + where subcommand is one of: +""" + title("Display") + + for l in LISTABLE: + usage += " list {0} [<column>...]\n".format(l) + for s in SHOWABLE: + usage += " show {0} [<column>...]\n".format(s) + usage += title("Object Manipulation") + usage += fmt_usage_stanza(DECLARABLE, 'declare') + usage += fmt_usage_stanza(DELETABLE, 'delete') + usage += fmt_usage_stanza(CLOSABLE, 'close') + usage += fmt_usage_stanza(PURGABLE, 'purge') + usage += title("Broker Definitions") + usage += """ export <file> + import <file> +""" + usage += title("Publishing and Consuming") + usage += fmt_usage_stanza(EXTRA_VERBS, '') + usage += """ + * If payload is not specified on publish, standard input is used + + * If payload_file is not specified on get, the payload will be shown on + standard output along with the message metadata + + * If payload_file is specified on get, count must not be set +""" + return usage + +def config_usage(): + usage = "Usage\n=====\n" + short_usage() + usage += "\n" + title("Configuration File") + usage += """ It is possible to specify a configuration file from the command line. + Hosts can be configured easily in a configuration file and called + from the command line. +""" + usage += title("Example") + usage += """ # rabbitmqadmin.conf.example START + + [host_normal] + hostname = localhost + port = 15672 + username = guest + password = guest + declare_vhost = / # Used as default for declare / delete only + vhost = / # Used as default for declare / delete / list + + [host_ssl] + hostname = otherhost + port = 15672 + username = guest + password = guest + ssl = True + ssl_key_file = /path/to/key.pem + ssl_cert_file = /path/to/cert.pem + + # rabbitmqadmin.conf.example END +""" + usage += title("Use") + usage += """ rabbitmqadmin -c rabbitmqadmin.conf.example -N host_normal ...""" + return usage + +def more_help(): + return """ +More Help +========= + +For more help use the help subcommand: + + rabbitmqadmin help subcommands # For a list of available subcommands + rabbitmqadmin help config # For help with the configuration file +""" + +def fmt_usage_stanza(root, verb): + def fmt_args(args): + res = " ".join(["{0}=...".format(a) for a in args['mandatory']]) + opts = " ".join("{0}=...".format(o) for o in args['optional'].keys()) + if opts != "": + res += " [{0}]".format(opts) + return res + + text = "" + if verb != "": + verb = " " + verb + for k in root.keys(): + text += " {0} {1} {2}\n".format(verb, k, fmt_args(root[k])) + return text + +default_options = { "hostname" : "localhost", + "port" : "15672", + "declare_vhost" : "/", + "username" : "guest", + "password" : "guest", + "ssl" : False, + "verbose" : True, + "format" : "table", + "depth" : 1, + "bash_completion" : False } + + +class MyFormatter(TitledHelpFormatter): + def format_epilog(self, epilog): + return epilog + +parser = OptionParser(usage=short_usage(), + formatter=MyFormatter(), + epilog=more_help()) + +def make_parser(): + def add(*args, **kwargs): + key = kwargs['dest'] + if key in default_options: + default = " [default: %s]" % default_options[key] + kwargs['help'] = kwargs['help'] + default + parser.add_option(*args, **kwargs) + + add("-c", "--config", dest="config", + help="configuration file [default: ~/.rabbitmqadmin.conf]", + metavar="CONFIG") + add("-N", "--node", dest="node", + help="node described in the configuration file [default: 'default'" + \ + " only if configuration file is specified]", + metavar="NODE") + add("-H", "--host", dest="hostname", + help="connect to host HOST" , + metavar="HOST") + add("-P", "--port", dest="port", + help="connect to port PORT", + metavar="PORT") + add("-V", "--vhost", dest="vhost", + help="connect to vhost VHOST [default: all vhosts for list, '/' for declare]", + metavar="VHOST") + add("-u", "--username", dest="username", + help="connect using username USERNAME", + metavar="USERNAME") + add("-p", "--password", dest="password", + help="connect using password PASSWORD", + metavar="PASSWORD") + add("-q", "--quiet", action="store_false", dest="verbose", + help="suppress status messages") + add("-s", "--ssl", action="store_true", dest="ssl", + help="connect with ssl") + add("--ssl-key-file", dest="ssl_key_file", + help="PEM format key file for SSL") + add("--ssl-cert-file", dest="ssl_cert_file", + help="PEM format certificate file for SSL") + add("-f", "--format", dest="format", + help="format for listing commands - one of [" + ", ".join(FORMATS.keys()) + "]") + add("-S", "--sort", dest="sort", help="sort key for listing queries") + add("-R", "--sort-reverse", action="store_true", dest="sort_reverse", + help="reverse the sort order") + add("-d", "--depth", dest="depth", + help="maximum depth to recurse for listing tables") + add("--bash-completion", action="store_true", + dest="bash_completion", + help="Print bash completion script") + add("--version", action="store_true", + dest="version", + help="Display version and exit") + +def default_config(): + home = os.getenv('USERPROFILE') or os.getenv('HOME') + if home is not None: + config_file = home + os.sep + ".rabbitmqadmin.conf" + if os.path.isfile(config_file): + return config_file + return None + +def make_configuration(): + make_parser() + (options, args) = parser.parse_args() + setattr(options, "declare_vhost", None) + if options.version: + print_version() + if options.config is None: + config_file = default_config() + if config_file is not None: + setattr(options, "config", config_file) + else: + if not os.path.isfile(options.config): + assert_usage(False, + "Could not read config file '%s'" % options.config) + + if options.node is None and options.config: + options.node = "default" + else: + options.node = options.node + for (key, val) in default_options.items(): + if getattr(options, key) is None: + setattr(options, key, val) + + if options.config is not None: + config = ConfigParser() + try: + config.read(options.config) + new_conf = dict(config.items(options.node)) + except NoSectionError, error: + if options.node == "default": + pass + else: + assert_usage(False, ("Could not read section '%s' in config file" + + " '%s':\n %s") % + (options.node, options.config, error)) + else: + for key, val in new_conf.items(): + setattr(options, key, val) + + return (options, args) + +def assert_usage(expr, error): + if not expr: + output("\nERROR: {0}\n".format(error)) + output("{0} --help for help\n".format(os.path.basename(sys.argv[0]))) + sys.exit(1) + +def print_version(): + output("rabbitmqadmin {0}".format(VERSION)) + sys.exit(0) + +def column_sort_key(col): + if col in PROMOTE_COLUMNS: + return (1, PROMOTE_COLUMNS.index(col)) + else: + return (2, col) + +def main(): + (options, args) = make_configuration() + if options.bash_completion: + print_bash_completion() + exit(0) + assert_usage(len(args) > 0, 'Action not specified') + mgmt = Management(options, args[1:]) + mode = "invoke_" + args[0] + assert_usage(hasattr(mgmt, mode), + 'Action {0} not understood'.format(args[0])) + method = getattr(mgmt, "invoke_%s" % args[0]) + method() + +def output(s): + print maybe_utf8(s, sys.stdout) + +def die(s): + sys.stderr.write(maybe_utf8("*** {0}\n".format(s), sys.stderr)) + exit(1) + +def maybe_utf8(s, stream): + if stream.isatty(): + # It will have an encoding, which Python will respect + return s + else: + # It won't have an encoding, and Python will pick ASCII by default + return s.encode('utf-8') + +class Management: + def __init__(self, options, args): + self.options = options + self.args = args + + def get(self, path): + return self.http("GET", "/api%s" % path, "") + + def put(self, path, body): + return self.http("PUT", "/api%s" % path, body) + + def post(self, path, body): + return self.http("POST", "/api%s" % path, body) + + def delete(self, path): + return self.http("DELETE", "/api%s" % path, "") + + def http(self, method, path, body): + if self.options.ssl: + conn = httplib.HTTPSConnection(self.options.hostname, + self.options.port, + self.options.ssl_key_file, + self.options.ssl_cert_file) + else: + conn = httplib.HTTPConnection(self.options.hostname, + self.options.port) + headers = {"Authorization": + "Basic " + base64.b64encode(self.options.username + ":" + + self.options.password)} + if body != "": + headers["Content-Type"] = "application/json" + try: + conn.request(method, path, body, headers) + except socket.error, e: + die("Could not connect: {0}".format(e)) + resp = conn.getresponse() + if resp.status == 400: + die(json.loads(resp.read())['reason']) + if resp.status == 401: + die("Access refused: {0}".format(path)) + if resp.status == 404: + die("Not found: {0}".format(path)) + if resp.status == 301: + url = urlparse.urlparse(resp.getheader('location')) + [host, port] = url.netloc.split(':') + self.options.hostname = host + self.options.port = int(port) + return self.http(method, url.path + '?' + url.query, body) + if resp.status < 200 or resp.status > 400: + raise Exception("Received %d %s for path %s\n%s" + % (resp.status, resp.reason, path, resp.read())) + return resp.read() + + def verbose(self, string): + if self.options.verbose: + output(string) + + def get_arg(self): + assert_usage(len(self.args) == 1, 'Exactly one argument required') + return self.args[0] + + def invoke_help(self): + if len(self.args) == 0: + parser.print_help() + else: + help_cmd = self.get_arg() + if help_cmd == 'subcommands': + usage = subcommands_usage() + elif help_cmd == 'config': + usage = config_usage() + else: + assert_usage(False, """help topic must be one of: + subcommands + config""") + print usage + exit(0) + + def invoke_publish(self): + (uri, upload) = self.parse_args(self.args, EXTRA_VERBS['publish']) + upload['properties'] = {} # TODO do we care here? + if not 'payload' in upload: + data = sys.stdin.read() + upload['payload'] = base64.b64encode(data) + upload['payload_encoding'] = 'base64' + resp = json.loads(self.post(uri, json.dumps(upload))) + if resp['routed']: + self.verbose("Message published") + else: + self.verbose("Message published but NOT routed") + + def invoke_get(self): + (uri, upload) = self.parse_args(self.args, EXTRA_VERBS['get']) + payload_file = 'payload_file' in upload and upload['payload_file'] or None + assert_usage(not payload_file or upload['count'] == '1', + 'Cannot get multiple messages using payload_file') + result = self.post(uri, json.dumps(upload)) + if payload_file: + write_payload_file(payload_file, result) + columns = ['routing_key', 'exchange', 'message_count', + 'payload_bytes', 'redelivered'] + format_list(result, columns, {}, self.options) + else: + format_list(result, [], {}, self.options) + + def invoke_export(self): + path = self.get_arg() + definitions = self.get("/definitions") + f = open(path, 'w') + f.write(definitions) + f.close() + self.verbose("Exported definitions for %s to \"%s\"" + % (self.options.hostname, path)) + + def invoke_import(self): + path = self.get_arg() + f = open(path, 'r') + definitions = f.read() + f.close() + self.post("/definitions", definitions) + self.verbose("Imported definitions for %s from \"%s\"" + % (self.options.hostname, path)) + + def invoke_list(self): + cols = self.args[1:] + (uri, obj_info) = self.list_show_uri(LISTABLE, 'list', cols) + format_list(self.get(uri), cols, obj_info, self.options) + + def invoke_show(self): + cols = self.args[1:] + (uri, obj_info) = self.list_show_uri(SHOWABLE, 'show', cols) + format_list('[{0}]'.format(self.get(uri)), cols, obj_info, self.options) + + def list_show_uri(self, obj_types, verb, cols): + obj_type = self.args[0] + assert_usage(obj_type in obj_types, + "Don't know how to {0} {1}".format(verb, obj_type)) + obj_info = obj_types[obj_type] + uri = "/%s" % obj_type + query = [] + if obj_info['vhost'] and self.options.vhost: + uri += "/%s" % urllib.quote_plus(self.options.vhost) + if cols != []: + query.append("columns=" + ",".join(cols)) + sort = self.options.sort + if sort: + query.append("sort=" + sort) + if self.options.sort_reverse: + query.append("sort_reverse=true") + query = "&".join(query) + if query != "": + uri += "?" + query + return (uri, obj_info) + + def invoke_declare(self): + (obj_type, uri, upload) = self.declare_delete_parse(DECLARABLE) + if obj_type == 'binding': + self.post(uri, json.dumps(upload)) + else: + self.put(uri, json.dumps(upload)) + self.verbose("{0} declared".format(obj_type)) + + def invoke_delete(self): + (obj_type, uri, upload) = self.declare_delete_parse(DELETABLE) + self.delete(uri) + self.verbose("{0} deleted".format(obj_type)) + + def invoke_close(self): + (obj_type, uri, upload) = self.declare_delete_parse(CLOSABLE) + self.delete(uri) + self.verbose("{0} closed".format(obj_type)) + + def invoke_purge(self): + (obj_type, uri, upload) = self.declare_delete_parse(PURGABLE) + self.delete(uri) + self.verbose("{0} purged".format(obj_type)) + + def declare_delete_parse(self, root): + assert_usage(len(self.args) > 0, 'Type not specified') + obj_type = self.args[0] + assert_usage(obj_type in root, + 'Type {0} not recognised'.format(obj_type)) + obj = root[obj_type] + (uri, upload) = self.parse_args(self.args[1:], obj) + return (obj_type, uri, upload) + + def parse_args(self, args, obj): + mandatory = obj['mandatory'] + optional = obj['optional'] + uri_template = obj['uri'] + upload = {} + for k in optional.keys(): + if optional[k]: + upload[k] = optional[k] + for arg in args: + assert_usage("=" in arg, + 'Argument "{0}" not in format name=value'.format(arg)) + (name, value) = arg.split("=", 1) + assert_usage(name in mandatory or name in optional.keys(), + 'Argument "{0}" not recognised'.format(name)) + if 'json' in obj and name in obj['json']: + upload[name] = self.parse_json(value) + else: + upload[name] = value + for m in mandatory: + assert_usage(m in upload.keys(), + 'mandatory argument "{0}" required'.format(m)) + if 'vhost' not in mandatory: + upload['vhost'] = self.options.vhost or self.options.declare_vhost + uri_args = {} + for k in upload: + v = upload[k] + if v and isinstance(v, basestring): + uri_args[k] = urllib.quote_plus(v) + if k == 'destination_type': + uri_args['destination_char'] = v[0] + uri = uri_template.format(**uri_args) + return (uri, upload) + + def parse_json(self, text): + try: + return json.loads(text) + except ValueError: + print "Could not parse JSON:\n {0}".format(text) + sys.exit(1) + +def format_list(json_list, columns, args, options): + format = options.format + formatter = None + if format == "raw_json": + output(json_list) + return + elif format == "pretty_json": + enc = json.JSONEncoder(False, False, True, True, True, 2) + output(enc.encode(json.loads(json_list))) + return + else: + formatter = FORMATS[format] + assert_usage(formatter != None, + "Format {0} not recognised".format(format)) + formatter_instance = formatter(columns, args, options) + formatter_instance.display(json_list) + +class Lister: + def verbose(self, string): + if self.options.verbose: + output(string) + + def display(self, json_list): + depth = sys.maxint + if len(self.columns) == 0: + depth = int(self.options.depth) + (columns, table) = self.list_to_table(json.loads(json_list), depth) + if len(table) > 0: + self.display_list(columns, table) + else: + self.verbose("No items") + + def list_to_table(self, items, max_depth): + columns = {} + column_ix = {} + row = None + table = [] + + def add(prefix, depth, item, fun): + for key in item: + column = prefix == '' and key or (prefix + '.' + key) + subitem = item[key] + if type(subitem) == dict: + if self.obj_info.has_key('json') and key in self.obj_info['json']: + fun(column, json.dumps(subitem)) + else: + if depth < max_depth: + add(column, depth + 1, subitem, fun) + elif type(subitem) == list: + # The first branch has mirror nodes in queues in + # mind (which come out looking decent); the second + # one has applications in nodes (which look less + # so, but what would look good?). + if [x for x in subitem if type(x) != unicode] == []: + serialised = " ".join(subitem) + else: + serialised = json.dumps(subitem) + fun(column, serialised) + else: + fun(column, subitem) + + def add_to_columns(col, val): + columns[col] = True + + def add_to_row(col, val): + if col in column_ix: + row[column_ix[col]] = unicode(val) + + if len(self.columns) == 0: + for item in items: + add('', 1, item, add_to_columns) + columns = columns.keys() + columns.sort(key=column_sort_key) + else: + columns = self.columns + + for i in xrange(0, len(columns)): + column_ix[columns[i]] = i + for item in items: + row = len(columns) * [''] + add('', 1, item, add_to_row) + table.append(row) + + return (columns, table) + +class TSVList(Lister): + def __init__(self, columns, obj_info, options): + self.columns = columns + self.obj_info = obj_info + self.options = options + + def display_list(self, columns, table): + head = "\t".join(columns) + self.verbose(head) + + for row in table: + line = "\t".join(row) + output(line) + +class LongList(Lister): + def __init__(self, columns, obj_info, options): + self.columns = columns + self.obj_info = obj_info + self.options = options + + def display_list(self, columns, table): + sep = "\n" + "-" * 80 + "\n" + max_width = 0 + for col in columns: + max_width = max(max_width, len(col)) + fmt = "{0:>" + unicode(max_width) + "}: {1}" + output(sep) + for i in xrange(0, len(table)): + for j in xrange(0, len(columns)): + output(fmt.format(columns[j], table[i][j])) + output(sep) + +class TableList(Lister): + def __init__(self, columns, obj_info, options): + self.columns = columns + self.obj_info = obj_info + self.options = options + + def display_list(self, columns, table): + total = [columns] + total.extend(table) + self.ascii_table(total) + + def ascii_table(self, rows): + table = "" + col_widths = [0] * len(rows[0]) + for i in xrange(0, len(rows[0])): + for j in xrange(0, len(rows)): + col_widths[i] = max(col_widths[i], len(rows[j][i])) + self.ascii_bar(col_widths) + self.ascii_row(col_widths, rows[0], "^") + self.ascii_bar(col_widths) + for row in rows[1:]: + self.ascii_row(col_widths, row, "<") + self.ascii_bar(col_widths) + + def ascii_row(self, col_widths, row, align): + txt = "|" + for i in xrange(0, len(col_widths)): + fmt = " {0:" + align + unicode(col_widths[i]) + "} " + txt += fmt.format(row[i]) + "|" + output(txt) + + def ascii_bar(self, col_widths): + txt = "+" + for w in col_widths: + txt += ("-" * (w + 2)) + "+" + output(txt) + +class KeyValueList(Lister): + def __init__(self, columns, obj_info, options): + self.columns = columns + self.obj_info = obj_info + self.options = options + + def display_list(self, columns, table): + for i in xrange(0, len(table)): + row = [] + for j in xrange(0, len(columns)): + row.append("{0}=\"{1}\"".format(columns[j], table[i][j])) + output(" ".join(row)) + +# TODO handle spaces etc in completable names +class BashList(Lister): + def __init__(self, columns, obj_info, options): + self.columns = columns + self.obj_info = obj_info + self.options = options + + def display_list(self, columns, table): + ix = None + for i in xrange(0, len(columns)): + if columns[i] == 'name': + ix = i + if ix is not None: + res = [] + for row in table: + res.append(row[ix]) + output(" ".join(res)) + +FORMATS = { + 'raw_json' : None, # Special cased + 'pretty_json' : None, # Ditto + 'tsv' : TSVList, + 'long' : LongList, + 'table' : TableList, + 'kvp' : KeyValueList, + 'bash' : BashList +} + +def write_payload_file(payload_file, json_list): + result = json.loads(json_list)[0] + payload = result['payload'] + payload_encoding = result['payload_encoding'] + f = open(payload_file, 'w') + if payload_encoding == 'base64': + data = base64.b64decode(payload) + else: + data = payload + f.write(data) + f.close() + +def print_bash_completion(): + script = """# This is a bash completion script for rabbitmqadmin. +# Redirect it to a file, then source it or copy it to /etc/bash_completion.d +# to get tab completion. rabbitmqadmin must be on your PATH for this to work. +_rabbitmqadmin() +{ + local cur prev opts base + COMPREPLY=() + cur="${COMP_WORDS[COMP_CWORD]}" + prev="${COMP_WORDS[COMP_CWORD-1]}" + + opts="list show declare delete close purge import export get publish help" + fargs="--help --host --port --vhost --username --password --format --depth --sort --sort-reverse" + + case "${prev}" in + list) + COMPREPLY=( $(compgen -W '""" + " ".join(LISTABLE) + """' -- ${cur}) ) + return 0 + ;; + show) + COMPREPLY=( $(compgen -W '""" + " ".join(SHOWABLE) + """' -- ${cur}) ) + return 0 + ;; + declare) + COMPREPLY=( $(compgen -W '""" + " ".join(DECLARABLE.keys()) + """' -- ${cur}) ) + return 0 + ;; + delete) + COMPREPLY=( $(compgen -W '""" + " ".join(DELETABLE.keys()) + """' -- ${cur}) ) + return 0 + ;; + close) + COMPREPLY=( $(compgen -W '""" + " ".join(CLOSABLE.keys()) + """' -- ${cur}) ) + return 0 + ;; + purge) + COMPREPLY=( $(compgen -W '""" + " ".join(PURGABLE.keys()) + """' -- ${cur}) ) + return 0 + ;; + export) + COMPREPLY=( $(compgen -f ${cur}) ) + return 0 + ;; + import) + COMPREPLY=( $(compgen -f ${cur}) ) + return 0 + ;; + help) + opts="subcommands config" + COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) + return 0 + ;; + -H) + COMPREPLY=( $(compgen -A hostname ${cur}) ) + return 0 + ;; + --host) + COMPREPLY=( $(compgen -A hostname ${cur}) ) + return 0 + ;; + -V) + opts="$(rabbitmqadmin -q -f bash list vhosts)" + COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) + return 0 + ;; + --vhost) + opts="$(rabbitmqadmin -q -f bash list vhosts)" + COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) + return 0 + ;; + -u) + opts="$(rabbitmqadmin -q -f bash list users)" + COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) + return 0 + ;; + --username) + opts="$(rabbitmqadmin -q -f bash list users)" + COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) + return 0 + ;; + -f) + COMPREPLY=( $(compgen -W \"""" + " ".join(FORMATS.keys()) + """\" -- ${cur}) ) + return 0 + ;; + --format) + COMPREPLY=( $(compgen -W \"""" + " ".join(FORMATS.keys()) + """\" -- ${cur}) ) + return 0 + ;; + +""" + for l in LISTABLE: + key = l[0:len(l) - 1] + script += " " + key + """) + opts="$(rabbitmqadmin -q -f bash list """ + l + """)" + COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) + return 0 + ;; +""" + script += """ *) + ;; + esac + + COMPREPLY=($(compgen -W "${opts} ${fargs}" -- ${cur})) + return 0 +} +complete -F _rabbitmqadmin rabbitmqadmin +""" + output(script) + +if __name__ == "__main__": + main() diff --git a/deps/rabbit/test/term_to_binary_compat_prop_SUITE.erl b/deps/rabbit/test/term_to_binary_compat_prop_SUITE.erl new file mode 100644 index 0000000000..2f56f56189 --- /dev/null +++ b/deps/rabbit/test/term_to_binary_compat_prop_SUITE.erl @@ -0,0 +1,105 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2017-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + + +-module(term_to_binary_compat_prop_SUITE). + +-compile(export_all). + +-include("rabbit.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("proper/include/proper.hrl"). + +-define(ITERATIONS_TO_RUN_UNTIL_CONFIDENT, 10000). + +all() -> + [ + ensure_term_to_binary_defaults_to_version_1, + term_to_binary_latin_atom, + queue_name_to_binary + ]. + +erts_gt_8() -> + Vsn = erlang:system_info(version), + [Maj|_] = string:tokens(Vsn, "."), + list_to_integer(Maj) > 8. + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_started(Config, Testcase). + +%% R16B03 defaults term_to_binary version to 0, this test would always fail +ensure_term_to_binary_defaults_to_version_1(Config) -> + CurrentERTS = erlang:system_info(version), + MinimumTestedERTS = "6.0", + case rabbit_misc:version_compare(CurrentERTS, MinimumTestedERTS, gte) of + true -> + Property = fun () -> + prop_ensure_term_to_binary_defaults_to_version_1(Config) + end, + rabbit_ct_proper_helpers:run_proper( + Property, [], + ?ITERATIONS_TO_RUN_UNTIL_CONFIDENT); + false -> + ct:pal( + ?LOW_IMPORTANCE, + "This test require ERTS ~p or above, running on ~p~n" + "Skipping test...", + [MinimumTestedERTS, CurrentERTS]) + end. + +prop_ensure_term_to_binary_defaults_to_version_1(_Config) -> + ?FORALL(Term, any(), + begin + Current = term_to_binary(Term), + Compat = term_to_binary_compat:term_to_binary_1(Term), + Current =:= Compat + end). + +term_to_binary_latin_atom(Config) -> + Property = fun () -> prop_term_to_binary_latin_atom(Config) end, + rabbit_ct_proper_helpers:run_proper(Property, [], + ?ITERATIONS_TO_RUN_UNTIL_CONFIDENT). + +prop_term_to_binary_latin_atom(_Config) -> + ?FORALL(LatinString, list(integer(0, 255)), + begin + Length = length(LatinString), + Atom = list_to_atom(LatinString), + Binary = list_to_binary(LatinString), + <<131,100, Length:16, Binary/binary>> =:= + term_to_binary_compat:term_to_binary_1(Atom) + end). + +queue_name_to_binary(Config) -> + Property = fun () -> prop_queue_name_to_binary(Config) end, + rabbit_ct_proper_helpers:run_proper(Property, [], + ?ITERATIONS_TO_RUN_UNTIL_CONFIDENT). + + +prop_queue_name_to_binary(_Config) -> + ?FORALL({VHost, QName}, {binary(), binary()}, + begin + VHostBSize = byte_size(VHost), + NameBSize = byte_size(QName), + Expected = + <<131, %% Binary format "version" + 104, 4, %% 4-element tuple + 100, 0, 8, "resource", %% `resource` atom + 109, VHostBSize:32, VHost/binary, %% Vhost binary + 100, 0, 5, "queue", %% `queue` atom + 109, NameBSize:32, QName/binary>>, %% Name binary + Resource = rabbit_misc:r(VHost, queue, QName), + Current = term_to_binary_compat:term_to_binary_1(Resource), + Current =:= Expected + end). diff --git a/deps/rabbit/test/test_util.erl b/deps/rabbit/test/test_util.erl new file mode 100644 index 0000000000..9a82b0ea1c --- /dev/null +++ b/deps/rabbit/test/test_util.erl @@ -0,0 +1,28 @@ +-module(test_util). + +-export([ + fake_pid/1 + ]). + + +fake_pid(Node) -> + NodeBin = rabbit_data_coercion:to_binary(Node), + ThisNodeSize = size(term_to_binary(node())) + 1, + Pid = spawn(fun () -> ok end), + %% drop the local node data from a local pid + <<Pre:ThisNodeSize/binary, LocalPidData/binary>> = term_to_binary(Pid), + S = size(NodeBin), + %% get the encoding type of the pid + <<_:8, Type:8/unsigned, _/binary>> = Pre, + %% replace it with the incoming node binary + Final = <<131, Type, 100, S:16/unsigned, NodeBin/binary, LocalPidData/binary>>, + binary_to_term(Final). + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). + +fake_pid_test() -> + _ = fake_pid(banana), + ok. + +-endif. diff --git a/deps/rabbit/test/topic_permission_SUITE.erl b/deps/rabbit/test/topic_permission_SUITE.erl new file mode 100644 index 0000000000..2f123fd7f6 --- /dev/null +++ b/deps/rabbit/test/topic_permission_SUITE.erl @@ -0,0 +1,244 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2011-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(topic_permission_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("rabbit_common/include/rabbit.hrl"). + +-compile(export_all). + +all() -> + [ + {group, sequential_tests} + ]. + +groups() -> [ + {sequential_tests, [], [ + topic_permission_database_access, + topic_permission_checks + ]} + ]. + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + Config1 = rabbit_ct_helpers:set_config(Config, [ + {rmq_nodename_suffix, ?MODULE} + ]), + rabbit_ct_helpers:run_setup_steps(Config1, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()). + +init_per_group(_, Config) -> Config. +end_per_group(_, Config) -> Config. + +init_per_testcase(Testcase, Config) -> + ok = rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, clear_tables, []), + rabbit_ct_helpers:testcase_started(Config, Testcase). + +clear_tables() -> + {atomic, ok} = mnesia:clear_table(rabbit_topic_permission), + {atomic, ok} = mnesia:clear_table(rabbit_vhost), + {atomic, ok} = mnesia:clear_table(rabbit_user), + ok. + +end_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_finished(Config, Testcase). + +topic_permission_database_access(Config) -> + ok = rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, topic_permission_database_access1, [Config]). + +topic_permission_database_access1(_Config) -> + 0 = length(ets:tab2list(rabbit_topic_permission)), + rabbit_vhost:add(<<"/">>, <<"acting-user">>), + rabbit_vhost:add(<<"other-vhost">>, <<"acting-user">>), + rabbit_auth_backend_internal:add_user(<<"guest">>, <<"guest">>, <<"acting-user">>), + rabbit_auth_backend_internal:add_user(<<"dummy">>, <<"dummy">>, <<"acting-user">>), + + rabbit_auth_backend_internal:set_topic_permissions( + <<"guest">>, <<"/">>, <<"amq.topic">>, "^a", "^a", <<"acting-user">> + ), + 1 = length(ets:tab2list(rabbit_topic_permission)), + 1 = length(rabbit_auth_backend_internal:list_user_topic_permissions(<<"guest">>)), + 0 = length(rabbit_auth_backend_internal:list_user_topic_permissions(<<"dummy">>)), + 1 = length(rabbit_auth_backend_internal:list_vhost_topic_permissions(<<"/">>)), + 0 = length(rabbit_auth_backend_internal:list_vhost_topic_permissions(<<"other-vhost">>)), + 1 = length(rabbit_auth_backend_internal:list_user_vhost_topic_permissions(<<"guest">>,<<"/">>)), + 0 = length(rabbit_auth_backend_internal:list_user_vhost_topic_permissions(<<"guest">>,<<"other-vhost">>)), + 1 = length(rabbit_auth_backend_internal:list_topic_permissions()), + + rabbit_auth_backend_internal:set_topic_permissions( + <<"guest">>, <<"other-vhost">>, <<"amq.topic">>, ".*", ".*", <<"acting-user">> + ), + 2 = length(ets:tab2list(rabbit_topic_permission)), + 2 = length(rabbit_auth_backend_internal:list_user_topic_permissions(<<"guest">>)), + 0 = length(rabbit_auth_backend_internal:list_user_topic_permissions(<<"dummy">>)), + 1 = length(rabbit_auth_backend_internal:list_vhost_topic_permissions(<<"/">>)), + 1 = length(rabbit_auth_backend_internal:list_vhost_topic_permissions(<<"other-vhost">>)), + 1 = length(rabbit_auth_backend_internal:list_user_vhost_topic_permissions(<<"guest">>,<<"/">>)), + 1 = length(rabbit_auth_backend_internal:list_user_vhost_topic_permissions(<<"guest">>,<<"other-vhost">>)), + 2 = length(rabbit_auth_backend_internal:list_topic_permissions()), + + rabbit_auth_backend_internal:set_topic_permissions( + <<"guest">>, <<"/">>, <<"topic1">>, "^a", "^a", <<"acting-user">> + ), + rabbit_auth_backend_internal:set_topic_permissions( + <<"guest">>, <<"/">>, <<"topic2">>, "^a", "^a", <<"acting-user">> + ), + + 4 = length(rabbit_auth_backend_internal:list_user_topic_permissions(<<"guest">>)), + 3 = length(rabbit_auth_backend_internal:list_user_vhost_topic_permissions(<<"guest">>,<<"/">>)), + 1 = length(rabbit_auth_backend_internal:list_user_vhost_topic_permissions(<<"guest">>,<<"other-vhost">>)), + 4 = length(rabbit_auth_backend_internal:list_topic_permissions()), + + rabbit_auth_backend_internal:clear_topic_permissions(<<"guest">>, <<"other-vhost">>, + <<"acting-user">>), + 0 = length(rabbit_auth_backend_internal:list_vhost_topic_permissions(<<"other-vhost">>)), + 3 = length(rabbit_auth_backend_internal:list_user_topic_permissions(<<"guest">>)), + rabbit_auth_backend_internal:clear_topic_permissions(<<"guest">>, <<"/">>, <<"topic1">>, + <<"acting-user">>), + 2 = length(rabbit_auth_backend_internal:list_user_topic_permissions(<<"guest">>)), + rabbit_auth_backend_internal:clear_topic_permissions(<<"guest">>, <<"/">>, + <<"acting-user">>), + 0 = length(rabbit_auth_backend_internal:list_user_topic_permissions(<<"guest">>)), + + + {error, {no_such_user, _}} = (catch rabbit_auth_backend_internal:set_topic_permissions( + <<"non-existing-user">>, <<"other-vhost">>, <<"amq.topic">>, ".*", ".*", <<"acting-user">> + )), + + {error, {no_such_vhost, _}} = (catch rabbit_auth_backend_internal:set_topic_permissions( + <<"guest">>, <<"non-existing-vhost">>, <<"amq.topic">>, ".*", ".*", <<"acting-user">> + )), + + {error, {no_such_user, _}} = (catch rabbit_auth_backend_internal:set_topic_permissions( + <<"non-existing-user">>, <<"non-existing-vhost">>, <<"amq.topic">>, ".*", ".*", <<"acting-user">> + )), + + {error, {no_such_user, _}} = (catch rabbit_auth_backend_internal:list_user_topic_permissions( + "non-existing-user" + )), + + {error, {no_such_vhost, _}} = (catch rabbit_auth_backend_internal:list_vhost_topic_permissions( + "non-existing-vhost" + )), + + {error, {invalid_regexp, _, _}} = (catch rabbit_auth_backend_internal:set_topic_permissions( + <<"guest">>, <<"/">>, <<"amq.topic">>, "[", "^a", <<"acting-user">> + )), + ok. + +topic_permission_checks(Config) -> + ok = rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, topic_permission_checks1, [Config]). + +topic_permission_checks1(_Config) -> + 0 = length(ets:tab2list(rabbit_topic_permission)), + rabbit_misc:execute_mnesia_transaction(fun() -> + ok = mnesia:write(rabbit_vhost, + vhost:new(<<"/">>, []), + write), + ok = mnesia:write(rabbit_vhost, + vhost:new(<<"other-vhost">>, []), + write) + end), + rabbit_auth_backend_internal:add_user(<<"guest">>, <<"guest">>, <<"acting-user">>), + rabbit_auth_backend_internal:add_user(<<"dummy">>, <<"dummy">>, <<"acting-user">>), + + rabbit_auth_backend_internal:set_topic_permissions( + <<"guest">>, <<"/">>, <<"amq.topic">>, "^a", "^a", <<"acting-user">> + ), + 1 = length(ets:tab2list(rabbit_topic_permission)), + 1 = length(rabbit_auth_backend_internal:list_user_topic_permissions(<<"guest">>)), + 0 = length(rabbit_auth_backend_internal:list_user_topic_permissions(<<"dummy">>)), + 1 = length(rabbit_auth_backend_internal:list_vhost_topic_permissions(<<"/">>)), + 0 = length(rabbit_auth_backend_internal:list_vhost_topic_permissions(<<"other-vhost">>)), + + rabbit_auth_backend_internal:set_topic_permissions( + <<"guest">>, <<"other-vhost">>, <<"amq.topic">>, ".*", ".*", <<"acting-user">> + ), + 2 = length(ets:tab2list(rabbit_topic_permission)), + 2 = length(rabbit_auth_backend_internal:list_user_topic_permissions(<<"guest">>)), + 0 = length(rabbit_auth_backend_internal:list_user_topic_permissions(<<"dummy">>)), + 1 = length(rabbit_auth_backend_internal:list_vhost_topic_permissions(<<"/">>)), + 1 = length(rabbit_auth_backend_internal:list_vhost_topic_permissions(<<"other-vhost">>)), + + User = #auth_user{username = <<"guest">>}, + Topic = #resource{name = <<"amq.topic">>, virtual_host = <<"/">>, + kind = topic}, + Context = #{routing_key => <<"a.b.c">>}, + Permissions = [write, read], + %% user has access to exchange, routing key matches + [true = rabbit_auth_backend_internal:check_topic_access( + User, + Topic, + Perm, + Context + ) || Perm <- Permissions], + %% user has access to exchange, routing key does not match + [false = rabbit_auth_backend_internal:check_topic_access( + User, + Topic, + Perm, + #{routing_key => <<"x.y.z">>} + ) || Perm <- Permissions], + %% user has access to exchange but not on this vhost + %% let pass when there's no match + [true = rabbit_auth_backend_internal:check_topic_access( + User, + Topic#resource{virtual_host = <<"fancyvhost">>}, + Perm, + Context + ) || Perm <- Permissions], + %% user does not have access to exchange + %% let pass when there's no match + [true = rabbit_auth_backend_internal:check_topic_access( + #auth_user{username = <<"dummy">>}, + Topic, + Perm, + Context + ) || Perm <- Permissions], + + %% expand variables + rabbit_auth_backend_internal:set_topic_permissions( + <<"guest">>, <<"other-vhost">>, <<"amq.topic">>, + "services.{vhost}.accounts.{username}.notifications", + "services.{vhost}.accounts.{username}.notifications", <<"acting-user">> + ), + %% routing key OK + [true = rabbit_auth_backend_internal:check_topic_access( + User, + Topic#resource{virtual_host = <<"other-vhost">>}, + Perm, + #{routing_key => <<"services.other-vhost.accounts.guest.notifications">>, + variable_map => #{ + <<"username">> => <<"guest">>, + <<"vhost">> => <<"other-vhost">> + } + } + ) || Perm <- Permissions], + %% routing key KO + [false = rabbit_auth_backend_internal:check_topic_access( + User, + Topic#resource{virtual_host = <<"other-vhost">>}, + Perm, + #{routing_key => <<"services.default.accounts.dummy.notifications">>, + variable_map => #{ + <<"username">> => <<"guest">>, + <<"vhost">> => <<"other-vhost">> + } + } + ) || Perm <- Permissions], + + ok. diff --git a/deps/rabbit/test/unit_access_control_SUITE.erl b/deps/rabbit/test/unit_access_control_SUITE.erl new file mode 100644 index 0000000000..af8f481083 --- /dev/null +++ b/deps/rabbit/test/unit_access_control_SUITE.erl @@ -0,0 +1,445 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2011-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(unit_access_control_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("kernel/include/file.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-compile(export_all). + +all() -> + [ + {group, sequential_tests}, + {group, parallel_tests} + ]. + +groups() -> + [ + {parallel_tests, [parallel], [ + password_hashing, + unsupported_connection_refusal + ]}, + {sequential_tests, [], [ + login_with_credentials_but_no_password, + login_of_passwordless_user, + set_tags_for_passwordless_user, + change_password, + topic_matching, + auth_backend_internal_expand_topic_permission, + rabbit_direct_extract_extra_auth_props + ]} + ]. + +%% ------------------------------------------------------------------- +%% Testsuite setup/teardown +%% ------------------------------------------------------------------- + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(Group, Config) -> + Config1 = rabbit_ct_helpers:set_config(Config, [ + {rmq_nodename_suffix, Group}, + {rmq_nodes_count, 1} + ]), + rabbit_ct_helpers:run_steps(Config1, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()). + +end_per_group(_Group, Config) -> + rabbit_ct_helpers:run_steps(Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()). + +init_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_started(Config, Testcase). + +end_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_finished(Config, Testcase). + +%% --------------------------------------------------------------------------- +%% Test Cases +%% --------------------------------------------------------------------------- + +password_hashing(Config) -> + passed = rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, password_hashing1, [Config]). + +password_hashing1(_Config) -> + rabbit_password_hashing_sha256 = rabbit_password:hashing_mod(), + application:set_env(rabbit, password_hashing_module, + rabbit_password_hashing_md5), + rabbit_password_hashing_md5 = rabbit_password:hashing_mod(), + application:set_env(rabbit, password_hashing_module, + rabbit_password_hashing_sha256), + rabbit_password_hashing_sha256 = rabbit_password:hashing_mod(), + + rabbit_password_hashing_sha256 = + rabbit_password:hashing_mod(rabbit_password_hashing_sha256), + rabbit_password_hashing_md5 = + rabbit_password:hashing_mod(rabbit_password_hashing_md5), + rabbit_password_hashing_md5 = + rabbit_password:hashing_mod(undefined), + + rabbit_password_hashing_md5 = + rabbit_auth_backend_internal:hashing_module_for_user( + internal_user:new()), + rabbit_password_hashing_md5 = + rabbit_auth_backend_internal:hashing_module_for_user( + internal_user:new({hashing_algorithm, undefined})), + rabbit_password_hashing_md5 = + rabbit_auth_backend_internal:hashing_module_for_user( + internal_user:new({hashing_algorithm, rabbit_password_hashing_md5})), + + rabbit_password_hashing_sha256 = + rabbit_auth_backend_internal:hashing_module_for_user( + internal_user:new({hashing_algorithm, rabbit_password_hashing_sha256})), + + passed. + +change_password(Config) -> + passed = rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, change_password1, [Config]). + +change_password1(_Config) -> + UserName = <<"test_user">>, + Password = <<"test_password">>, + case rabbit_auth_backend_internal:lookup_user(UserName) of + {ok, _} -> rabbit_auth_backend_internal:delete_user(UserName, <<"acting-user">>); + _ -> ok + end, + ok = application:set_env(rabbit, password_hashing_module, + rabbit_password_hashing_md5), + ok = rabbit_auth_backend_internal:add_user(UserName, Password, <<"acting-user">>), + {ok, #auth_user{username = UserName}} = + rabbit_auth_backend_internal:user_login_authentication( + UserName, [{password, Password}]), + ok = application:set_env(rabbit, password_hashing_module, + rabbit_password_hashing_sha256), + {ok, #auth_user{username = UserName}} = + rabbit_auth_backend_internal:user_login_authentication( + UserName, [{password, Password}]), + + NewPassword = <<"test_password1">>, + ok = rabbit_auth_backend_internal:change_password(UserName, NewPassword, + <<"acting-user">>), + {ok, #auth_user{username = UserName}} = + rabbit_auth_backend_internal:user_login_authentication( + UserName, [{password, NewPassword}]), + + {refused, _, [UserName]} = + rabbit_auth_backend_internal:user_login_authentication( + UserName, [{password, Password}]), + passed. + + +login_with_credentials_but_no_password(Config) -> + passed = rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, login_with_credentials_but_no_password1, [Config]). + +login_with_credentials_but_no_password1(_Config) -> + Username = <<"login_with_credentials_but_no_password-user">>, + Password = <<"login_with_credentials_but_no_password-password">>, + ok = rabbit_auth_backend_internal:add_user(Username, Password, <<"acting-user">>), + + try + rabbit_auth_backend_internal:user_login_authentication(Username, + [{key, <<"value">>}]), + ?assert(false) + catch exit:{unknown_auth_props, Username, [{key, <<"value">>}]} -> + ok + end, + + ok = rabbit_auth_backend_internal:delete_user(Username, <<"acting-user">>), + + passed. + +%% passwordless users are not supposed to be used with +%% this backend (and PLAIN authentication mechanism in general) +login_of_passwordless_user(Config) -> + passed = rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, login_of_passwordless_user1, [Config]). + +login_of_passwordless_user1(_Config) -> + Username = <<"login_of_passwordless_user-user">>, + Password = <<"">>, + ok = rabbit_auth_backend_internal:add_user(Username, Password, <<"acting-user">>), + + ?assertMatch( + {refused, _Message, [Username]}, + rabbit_auth_backend_internal:user_login_authentication(Username, + [{password, <<"">>}])), + + ?assertMatch( + {refused, _Format, [Username]}, + rabbit_auth_backend_internal:user_login_authentication(Username, + [{password, ""}])), + + ok = rabbit_auth_backend_internal:delete_user(Username, <<"acting-user">>), + + passed. + + +set_tags_for_passwordless_user(Config) -> + passed = rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, set_tags_for_passwordless_user1, [Config]). + +set_tags_for_passwordless_user1(_Config) -> + Username = <<"set_tags_for_passwordless_user">>, + Password = <<"set_tags_for_passwordless_user">>, + ok = rabbit_auth_backend_internal:add_user(Username, Password, + <<"acting-user">>), + ok = rabbit_auth_backend_internal:clear_password(Username, + <<"acting-user">>), + ok = rabbit_auth_backend_internal:set_tags(Username, [management], + <<"acting-user">>), + + {ok, User1} = rabbit_auth_backend_internal:lookup_user(Username), + ?assertEqual([management], internal_user:get_tags(User1)), + + ok = rabbit_auth_backend_internal:set_tags(Username, [management, policymaker], + <<"acting-user">>), + + {ok, User2} = rabbit_auth_backend_internal:lookup_user(Username), + ?assertEqual([management, policymaker], internal_user:get_tags(User2)), + + ok = rabbit_auth_backend_internal:set_tags(Username, [], + <<"acting-user">>), + + {ok, User3} = rabbit_auth_backend_internal:lookup_user(Username), + ?assertEqual([], internal_user:get_tags(User3)), + + ok = rabbit_auth_backend_internal:delete_user(Username, + <<"acting-user">>), + + passed. + + +rabbit_direct_extract_extra_auth_props(_Config) -> + {ok, CSC} = code_server_cache:start_link(), + % no protocol to extract + [] = rabbit_direct:extract_extra_auth_props( + {<<"guest">>, <<"guest">>}, <<"/">>, 1, + [{name,<<"127.0.0.1:52366 -> 127.0.0.1:1883">>}]), + % protocol to extract, but no module to call + [] = rabbit_direct:extract_extra_auth_props( + {<<"guest">>, <<"guest">>}, <<"/">>, 1, + [{protocol, {'PROTOCOL_WITHOUT_MODULE', "1.0"}}]), + % see rabbit_dummy_protocol_connection_info module + % protocol to extract, module that returns a client ID + [{client_id, <<"DummyClientId">>}] = rabbit_direct:extract_extra_auth_props( + {<<"guest">>, <<"guest">>}, <<"/">>, 1, + [{protocol, {'DUMMY_PROTOCOL', "1.0"}}]), + % protocol to extract, but error thrown in module + [] = rabbit_direct:extract_extra_auth_props( + {<<"guest">>, <<"guest">>}, <<"/">>, -1, + [{protocol, {'DUMMY_PROTOCOL', "1.0"}}]), + gen_server:stop(CSC), + ok. + +auth_backend_internal_expand_topic_permission(_Config) -> + ExpandMap = #{<<"username">> => <<"guest">>, <<"vhost">> => <<"default">>}, + %% simple case + <<"services/default/accounts/guest/notifications">> = + rabbit_auth_backend_internal:expand_topic_permission( + <<"services/{vhost}/accounts/{username}/notifications">>, + ExpandMap + ), + %% replace variable twice + <<"services/default/accounts/default/guest/notifications">> = + rabbit_auth_backend_internal:expand_topic_permission( + <<"services/{vhost}/accounts/{vhost}/{username}/notifications">>, + ExpandMap + ), + %% nothing to replace + <<"services/accounts/notifications">> = + rabbit_auth_backend_internal:expand_topic_permission( + <<"services/accounts/notifications">>, + ExpandMap + ), + %% the expand map isn't defined + <<"services/{vhost}/accounts/{username}/notifications">> = + rabbit_auth_backend_internal:expand_topic_permission( + <<"services/{vhost}/accounts/{username}/notifications">>, + undefined + ), + %% the expand map is empty + <<"services/{vhost}/accounts/{username}/notifications">> = + rabbit_auth_backend_internal:expand_topic_permission( + <<"services/{vhost}/accounts/{username}/notifications">>, + #{} + ), + ok. + +unsupported_connection_refusal(Config) -> + passed = rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, unsupported_connection_refusal1, [Config]). + +unsupported_connection_refusal1(Config) -> + H = ?config(rmq_hostname, Config), + P = rabbit_ct_broker_helpers:get_node_config(Config, 0, tcp_port_amqp), + [passed = test_unsupported_connection_refusal(H, P, V) || + V <- [<<"AMQP",9,9,9,9>>, <<"AMQP",0,1,0,0>>, <<"XXXX",0,0,9,1>>]], + passed. + +test_unsupported_connection_refusal(H, P, Header) -> + {ok, C} = gen_tcp:connect(H, P, [binary, {active, false}]), + ok = gen_tcp:send(C, Header), + {ok, <<"AMQP",0,0,9,1>>} = gen_tcp:recv(C, 8, 100), + ok = gen_tcp:close(C), + passed. + + +%% ------------------------------------------------------------------- +%% Topic matching. +%% ------------------------------------------------------------------- + +topic_matching(Config) -> + passed = rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, topic_matching1, [Config]). + +topic_matching1(_Config) -> + XName = #resource{virtual_host = <<"/">>, + kind = exchange, + name = <<"topic_matching-exchange">>}, + X0 = #exchange{name = XName, type = topic, durable = false, + auto_delete = false, arguments = []}, + X = rabbit_exchange_decorator:set(X0), + %% create + rabbit_exchange_type_topic:validate(X), + exchange_op_callback(X, create, []), + + %% add some bindings + Bindings = [#binding{source = XName, + key = list_to_binary(Key), + destination = #resource{virtual_host = <<"/">>, + kind = queue, + name = list_to_binary(Q)}, + args = Args} || + {Key, Q, Args} <- [{"a.b.c", "t1", []}, + {"a.*.c", "t2", []}, + {"a.#.b", "t3", []}, + {"a.b.b.c", "t4", []}, + {"#", "t5", []}, + {"#.#", "t6", []}, + {"#.b", "t7", []}, + {"*.*", "t8", []}, + {"a.*", "t9", []}, + {"*.b.c", "t10", []}, + {"a.#", "t11", []}, + {"a.#.#", "t12", []}, + {"b.b.c", "t13", []}, + {"a.b.b", "t14", []}, + {"a.b", "t15", []}, + {"b.c", "t16", []}, + {"", "t17", []}, + {"*.*.*", "t18", []}, + {"vodka.martini", "t19", []}, + {"a.b.c", "t20", []}, + {"*.#", "t21", []}, + {"#.*.#", "t22", []}, + {"*.#.#", "t23", []}, + {"#.#.#", "t24", []}, + {"*", "t25", []}, + {"#.b.#", "t26", []}, + {"args-test", "t27", + [{<<"foo">>, longstr, <<"bar">>}]}, + {"args-test", "t27", %% Note aliasing + [{<<"foo">>, longstr, <<"baz">>}]}]], + lists:foreach(fun (B) -> exchange_op_callback(X, add_binding, [B]) end, + Bindings), + + %% test some matches + test_topic_expect_match( + X, [{"a.b.c", ["t1", "t2", "t5", "t6", "t10", "t11", "t12", + "t18", "t20", "t21", "t22", "t23", "t24", + "t26"]}, + {"a.b", ["t3", "t5", "t6", "t7", "t8", "t9", "t11", + "t12", "t15", "t21", "t22", "t23", "t24", + "t26"]}, + {"a.b.b", ["t3", "t5", "t6", "t7", "t11", "t12", "t14", + "t18", "t21", "t22", "t23", "t24", "t26"]}, + {"", ["t5", "t6", "t17", "t24"]}, + {"b.c.c", ["t5", "t6", "t18", "t21", "t22", "t23", + "t24", "t26"]}, + {"a.a.a.a.a", ["t5", "t6", "t11", "t12", "t21", "t22", + "t23", "t24"]}, + {"vodka.gin", ["t5", "t6", "t8", "t21", "t22", "t23", + "t24"]}, + {"vodka.martini", ["t5", "t6", "t8", "t19", "t21", "t22", "t23", + "t24"]}, + {"b.b.c", ["t5", "t6", "t10", "t13", "t18", "t21", + "t22", "t23", "t24", "t26"]}, + {"nothing.here.at.all", ["t5", "t6", "t21", "t22", "t23", "t24"]}, + {"oneword", ["t5", "t6", "t21", "t22", "t23", "t24", + "t25"]}, + {"args-test", ["t5", "t6", "t21", "t22", "t23", "t24", + "t25", "t27"]}]), + %% remove some bindings + RemovedBindings = [lists:nth(1, Bindings), lists:nth(5, Bindings), + lists:nth(11, Bindings), lists:nth(19, Bindings), + lists:nth(21, Bindings), lists:nth(28, Bindings)], + exchange_op_callback(X, remove_bindings, [RemovedBindings]), + RemainingBindings = ordsets:to_list( + ordsets:subtract(ordsets:from_list(Bindings), + ordsets:from_list(RemovedBindings))), + + %% test some matches + test_topic_expect_match( + X, + [{"a.b.c", ["t2", "t6", "t10", "t12", "t18", "t20", "t22", + "t23", "t24", "t26"]}, + {"a.b", ["t3", "t6", "t7", "t8", "t9", "t12", "t15", + "t22", "t23", "t24", "t26"]}, + {"a.b.b", ["t3", "t6", "t7", "t12", "t14", "t18", "t22", + "t23", "t24", "t26"]}, + {"", ["t6", "t17", "t24"]}, + {"b.c.c", ["t6", "t18", "t22", "t23", "t24", "t26"]}, + {"a.a.a.a.a", ["t6", "t12", "t22", "t23", "t24"]}, + {"vodka.gin", ["t6", "t8", "t22", "t23", "t24"]}, + {"vodka.martini", ["t6", "t8", "t22", "t23", "t24"]}, + {"b.b.c", ["t6", "t10", "t13", "t18", "t22", "t23", + "t24", "t26"]}, + {"nothing.here.at.all", ["t6", "t22", "t23", "t24"]}, + {"oneword", ["t6", "t22", "t23", "t24", "t25"]}, + {"args-test", ["t6", "t22", "t23", "t24", "t25", "t27"]}]), + + %% remove the entire exchange + exchange_op_callback(X, delete, [RemainingBindings]), + %% none should match now + test_topic_expect_match(X, [{"a.b.c", []}, {"b.b.c", []}, {"", []}]), + passed. + +exchange_op_callback(X, Fun, Args) -> + rabbit_misc:execute_mnesia_transaction( + fun () -> rabbit_exchange:callback(X, Fun, transaction, [X] ++ Args) end), + rabbit_exchange:callback(X, Fun, none, [X] ++ Args). + +test_topic_expect_match(X, List) -> + lists:foreach( + fun ({Key, Expected}) -> + BinKey = list_to_binary(Key), + Message = rabbit_basic:message(X#exchange.name, BinKey, + #'P_basic'{}, <<>>), + Res = rabbit_exchange_type_topic:route( + X, #delivery{mandatory = false, + sender = self(), + message = Message}), + ExpectedRes = lists:map( + fun (Q) -> #resource{virtual_host = <<"/">>, + kind = queue, + name = list_to_binary(Q)} + end, Expected), + true = (lists:usort(ExpectedRes) =:= lists:usort(Res)) + end, List). diff --git a/deps/rabbit/test/unit_access_control_authn_authz_context_propagation_SUITE.erl b/deps/rabbit/test/unit_access_control_authn_authz_context_propagation_SUITE.erl new file mode 100644 index 0000000000..9cb1ad7267 --- /dev/null +++ b/deps/rabbit/test/unit_access_control_authn_authz_context_propagation_SUITE.erl @@ -0,0 +1,127 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2019-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(unit_access_control_authn_authz_context_propagation_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). + +-compile(export_all). + +all() -> + [ + {group, non_parallel_tests} + ]. + +groups() -> + [ + {non_parallel_tests, [], [ + propagate_context_to_auth_backend + ]} + ]. + +%% ------------------------------------------------------------------- +%% Testsuite setup/teardown. +%% ------------------------------------------------------------------- + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(_, Config) -> + Config. + +end_per_group(_, Config) -> + Config. + +init_per_testcase(Testcase, Config) -> + AuthConfig = {rabbit, [ + {auth_backends, [rabbit_auth_backend_context_propagation_mock]} + ] + }, + rabbit_ct_helpers:testcase_started(Config, Testcase), + Config1 = rabbit_ct_helpers:set_config(Config, [ + {rmq_nodename_suffix, Testcase} + ]), + rabbit_ct_helpers:run_setup_steps(Config1, + [ fun(Conf) -> merge_app_env(AuthConfig, Conf) end ] ++ + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()). + +merge_app_env(SomeConfig, Config) -> + rabbit_ct_helpers:merge_app_env(Config, SomeConfig). + +end_per_testcase(Testcase, Config) -> + Config1 = rabbit_ct_helpers:run_steps(Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()), + rabbit_ct_helpers:testcase_finished(Config1, Testcase). + +%% ------------------------------------------------------------------- +%% Test cases +%% ------------------------------------------------------------------- + +propagate_context_to_auth_backend(Config) -> + ok = rabbit_ct_broker_helpers:add_code_path_to_all_nodes(Config, + rabbit_auth_backend_context_propagation_mock), + passed = rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, propagate_context_to_auth_backend1, []). + +propagate_context_to_auth_backend1() -> + rabbit_auth_backend_context_propagation_mock:init(), + AmqpParams = #amqp_params_direct{ + virtual_host = <<"/">>, + username = <<"guest">>, + password = <<"guest">>, + adapter_info = #amqp_adapter_info{additional_info = [ + {variable_map, #{<<"key1">> => <<"value1">>}} + ], + protocol = {'FOO_PROTOCOL', '1.0'} %% this will trigger a call to rabbit_foo_protocol_connection_info + } + }, + {ok, Conn} = amqp_connection:start(AmqpParams), + + %% rabbit_direct will call the rabbit_foo_protocol_connection_info module to extract information + %% this information will be propagated to the authentication backend + [{authentication, AuthProps}] = rabbit_auth_backend_context_propagation_mock:get(authentication), + ?assertEqual(<<"value1">>, proplists:get_value(key1, AuthProps)), + + %% variable_map is propagated from rabbit_direct to the authorization backend + [{vhost_access, AuthzData}] = rabbit_auth_backend_context_propagation_mock:get(vhost_access), + ?assertEqual(<<"value1">>, maps:get(<<"key1">>, AuthzData)), + + %% variable_map is extracted when the channel is created and kept in its state + {ok, Ch} = amqp_connection:open_channel(Conn), + QName = <<"channel_propagate_context_to_authz_backend-q">>, + amqp_channel:call(Ch, #'queue.declare'{queue = QName}), + + check_send_receive(Ch, <<"">>, QName, QName), + amqp_channel:call(Ch, #'queue.bind'{queue = QName, exchange = <<"amq.topic">>, routing_key = <<"a.b">>}), + %% variable_map content is propagated from rabbit_channel to the authorization backend (resource check) + [{resource_access, AuthzContext}] = rabbit_auth_backend_context_propagation_mock:get(resource_access), + ?assertEqual(<<"value1">>, maps:get(<<"key1">>, AuthzContext)), + + check_send_receive(Ch, <<"amq.topic">>, <<"a.b">>, QName), + %% variable_map is propagated from rabbit_channel to the authorization backend (topic check) + [{topic_access, TopicContext}] = rabbit_auth_backend_context_propagation_mock:get(topic_access), + VariableMap = maps:get(variable_map, TopicContext), + ?assertEqual(<<"value1">>, maps:get(<<"key1">>, VariableMap)), + + passed. + +check_send_receive(Ch, Exchange, RoutingKey, QName) -> + amqp_channel:call(Ch, + #'basic.publish'{exchange = Exchange, routing_key = RoutingKey}, + #amqp_msg{payload = <<"foo">>}), + + {#'basic.get_ok'{}, #amqp_msg{payload = <<"foo">>}} = + amqp_channel:call(Ch, #'basic.get'{queue = QName, + no_ack = true}). diff --git a/deps/rabbit/test/unit_access_control_credential_validation_SUITE.erl b/deps/rabbit/test/unit_access_control_credential_validation_SUITE.erl new file mode 100644 index 0000000000..6a6a07836c --- /dev/null +++ b/deps/rabbit/test/unit_access_control_credential_validation_SUITE.erl @@ -0,0 +1,269 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(unit_access_control_credential_validation_SUITE). + +-compile(export_all). +-include_lib("proper/include/proper.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +all() -> + [ + {group, unit}, + {group, integration} + ]. + +groups() -> + [ + {integration, [], [ + min_length_integration_fails + , regexp_integration_fails + , min_length_integration_succeeds + , regexp_integration_succeeds + , min_length_change_password_integration_fails + , regexp_change_password_integration_fails + , min_length_change_password_integration_succeeds + , regexp_change_password_integration_succeeds + ]}, + {unit, [parallel], [ + basic_unconditionally_accepting_succeeds, + min_length_fails, + min_length_succeeds, + min_length_proper_fails, + min_length_proper_succeeds, + regexp_fails, + regexp_succeeds, + regexp_proper_fails, + regexp_proper_succeeds + ]} +]. + +suite() -> + [ + {timetrap, {minutes, 4}} + ]. + +%% +%% Setup/teardown +%% + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(integration, Config) -> + Suffix = rabbit_ct_helpers:testcase_absname(Config, "", "-"), + Config1 = rabbit_ct_helpers:set_config(Config, [ + {rmq_nodes_count, 1}, + {rmq_nodename_suffix, Suffix} + ]), + rabbit_ct_helpers:run_steps(Config1, + rabbit_ct_broker_helpers:setup_steps()); + +init_per_group(unit, Config) -> + Config. + +end_per_group(integration, Config) -> + rabbit_ct_broker_helpers:switch_credential_validator(Config, accept_everything), + rabbit_ct_helpers:run_steps(Config, + rabbit_ct_broker_helpers:teardown_steps()); +end_per_group(unit, Config) -> + Config. + +-define(USERNAME, <<"abc">>). + +init_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_started(Config, Testcase). + +end_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_finished(Config, Testcase). + +%% +%% Test Cases +%% + +basic_unconditionally_accepting_succeeds(_Config) -> + F = fun rabbit_credential_validator_accept_everything:validate/2, + + Pwd1 = crypto:strong_rand_bytes(1), + ?assertEqual(ok, F(?USERNAME, Pwd1)), + Pwd2 = crypto:strong_rand_bytes(5), + ?assertEqual(ok, F(?USERNAME, Pwd2)), + Pwd3 = crypto:strong_rand_bytes(10), + ?assertEqual(ok, F(?USERNAME, Pwd3)), + Pwd4 = crypto:strong_rand_bytes(50), + ?assertEqual(ok, F(?USERNAME, Pwd4)), + Pwd5 = crypto:strong_rand_bytes(100), + ?assertEqual(ok, F(?USERNAME, Pwd5)), + Pwd6 = crypto:strong_rand_bytes(1000), + ?assertEqual(ok, F(?USERNAME, Pwd6)). + +min_length_fails(_Config) -> + F = fun rabbit_credential_validator_min_password_length:validate/3, + + Pwd1 = crypto:strong_rand_bytes(1), + ?assertMatch({error, _}, F(?USERNAME, Pwd1, 5)), + Pwd2 = crypto:strong_rand_bytes(5), + ?assertMatch({error, _}, F(?USERNAME, Pwd2, 6)), + Pwd3 = crypto:strong_rand_bytes(10), + ?assertMatch({error, _}, F(?USERNAME, Pwd3, 15)), + Pwd4 = crypto:strong_rand_bytes(50), + ?assertMatch({error, _}, F(?USERNAME, Pwd4, 60)), + Pwd5 = undefined, + ?assertMatch({error, _}, F(?USERNAME, Pwd5, 60)), + Pwd6 = <<"">>, + ?assertMatch({error, _}, F(?USERNAME, Pwd6, 60)). + +min_length_succeeds(_Config) -> + F = fun rabbit_credential_validator_min_password_length:validate/3, + + ?assertEqual(ok, F(?USERNAME, crypto:strong_rand_bytes(1), 1)), + ?assertEqual(ok, F(?USERNAME, crypto:strong_rand_bytes(6), 6)), + ?assertEqual(ok, F(?USERNAME, crypto:strong_rand_bytes(7), 6)), + ?assertEqual(ok, F(?USERNAME, crypto:strong_rand_bytes(20), 20)), + ?assertEqual(ok, F(?USERNAME, crypto:strong_rand_bytes(40), 30)), + ?assertEqual(ok, F(?USERNAME, crypto:strong_rand_bytes(50), 50)). + +min_length_proper_fails(_Config) -> + rabbit_ct_proper_helpers:run_proper(fun prop_min_length_fails_validation/0, [], 500). + +min_length_proper_succeeds(_Config) -> + rabbit_ct_proper_helpers:run_proper(fun prop_min_length_passes_validation/0, [], 500). + +regexp_fails(_Config) -> + F = fun rabbit_credential_validator_password_regexp:validate/3, + + ?assertMatch({error, _}, F(?USERNAME, <<"abc">>, "^xyz")), + ?assertMatch({error, _}, F(?USERNAME, <<"abcdef">>, "^xyz")), + ?assertMatch({error, _}, F(?USERNAME, <<"abcxyz">>, "^abc\\d+")). + +regexp_succeeds(_Config) -> + F = fun rabbit_credential_validator_password_regexp:validate/3, + + ?assertEqual(ok, F(?USERNAME, <<"abc">>, "^abc")), + ?assertEqual(ok, F(?USERNAME, <<"abcdef">>, "^abc")), + ?assertEqual(ok, F(?USERNAME, <<"abc123">>, "^abc\\d+")). + +regexp_proper_fails(_Config) -> + rabbit_ct_proper_helpers:run_proper(fun prop_regexp_fails_validation/0, [], 500). + +regexp_proper_succeeds(_Config) -> + rabbit_ct_proper_helpers:run_proper(fun prop_regexp_passes_validation/0, [], 500). + +min_length_integration_fails(Config) -> + rabbit_ct_broker_helpers:delete_user(Config, ?USERNAME), + rabbit_ct_broker_helpers:switch_credential_validator(Config, min_length, 50), + ?assertMatch(rabbit_credential_validator_min_password_length, validator_backend(Config)), + ?assertMatch({error, "minimum required password length is 50"}, + rabbit_ct_broker_helpers:add_user(Config, ?USERNAME, <<"_">>)). + +regexp_integration_fails(Config) -> + rabbit_ct_broker_helpers:delete_user(Config, ?USERNAME), + rabbit_ct_broker_helpers:switch_credential_validator(Config, regexp), + ?assertMatch(rabbit_credential_validator_password_regexp, validator_backend(Config)), + ?assertMatch({error, _}, rabbit_ct_broker_helpers:add_user(Config, ?USERNAME, <<"_">>)). + +min_length_integration_succeeds(Config) -> + rabbit_ct_broker_helpers:delete_user(Config, ?USERNAME), + rabbit_ct_broker_helpers:switch_credential_validator(Config, min_length, 5), + ?assertMatch(rabbit_credential_validator_min_password_length, validator_backend(Config)), + ?assertMatch(ok, rabbit_ct_broker_helpers:add_user(Config, ?USERNAME, <<"abcdefghi">>)). + +regexp_integration_succeeds(Config) -> + rabbit_ct_broker_helpers:delete_user(Config, ?USERNAME), + rabbit_ct_broker_helpers:switch_credential_validator(Config, regexp), + ?assertMatch(rabbit_credential_validator_password_regexp, validator_backend(Config)), + ?assertMatch(ok, rabbit_ct_broker_helpers:add_user(Config, ?USERNAME, <<"xyz12345678901">>)). + +min_length_change_password_integration_fails(Config) -> + rabbit_ct_broker_helpers:delete_user(Config, ?USERNAME), + rabbit_ct_broker_helpers:switch_credential_validator(Config, accept_everything), + rabbit_ct_broker_helpers:add_user(Config, ?USERNAME, <<"abcdefghi">>), + rabbit_ct_broker_helpers:switch_credential_validator(Config, min_length, 50), + ?assertMatch(rabbit_credential_validator_min_password_length, validator_backend(Config)), + ?assertMatch({error, "minimum required password length is 50"}, + rabbit_ct_broker_helpers:change_password(Config, ?USERNAME, <<"_">>)). + +regexp_change_password_integration_fails(Config) -> + rabbit_ct_broker_helpers:delete_user(Config, ?USERNAME), + rabbit_ct_broker_helpers:switch_credential_validator(Config, accept_everything), + rabbit_ct_broker_helpers:add_user(Config, ?USERNAME, <<"abcdefghi">>), + rabbit_ct_broker_helpers:switch_credential_validator(Config, regexp), + ?assertMatch(rabbit_credential_validator_password_regexp, validator_backend(Config)), + ?assertMatch({error, _}, rabbit_ct_broker_helpers:change_password(Config, ?USERNAME, <<"_">>)). + +min_length_change_password_integration_succeeds(Config) -> + rabbit_ct_broker_helpers:delete_user(Config, ?USERNAME), + rabbit_ct_broker_helpers:switch_credential_validator(Config, accept_everything), + rabbit_ct_broker_helpers:add_user(Config, ?USERNAME, <<"abcdefghi">>), + rabbit_ct_broker_helpers:switch_credential_validator(Config, min_length, 5), + ?assertMatch(rabbit_credential_validator_min_password_length, validator_backend(Config)), + ?assertMatch(ok, rabbit_ct_broker_helpers:change_password(Config, ?USERNAME, <<"abcdefghi">>)). + +regexp_change_password_integration_succeeds(Config) -> + rabbit_ct_broker_helpers:delete_user(Config, ?USERNAME), + rabbit_ct_broker_helpers:switch_credential_validator(Config, accept_everything), + rabbit_ct_broker_helpers:add_user(Config, ?USERNAME, <<"abcdefghi">>), + rabbit_ct_broker_helpers:switch_credential_validator(Config, regexp), + ?assertMatch(rabbit_credential_validator_password_regexp, validator_backend(Config)), + ?assertMatch(ok, rabbit_ct_broker_helpers:change_password(Config, ?USERNAME, <<"xyz12345678901">>)). + +%% +%% PropEr +%% + +prop_min_length_fails_validation() -> + N = 5, + F = fun rabbit_credential_validator_min_password_length:validate/3, + ?FORALL(Val, binary(N), + ?FORALL(Length, choose(N + 1, 100), + failed_validation(F(?USERNAME, Val, Length + 1)))). + +prop_min_length_passes_validation() -> + N = 20, + F = fun rabbit_credential_validator_min_password_length:validate/3, + ?FORALL(Val, binary(N), + ?FORALL(Length, choose(1, N - 1), + passed_validation(F(?USERNAME, Val, Length)))). + +prop_regexp_fails_validation() -> + N = 5, + F = fun rabbit_credential_validator_password_regexp:validate/3, + ?FORALL(Val, binary(N), + ?FORALL(Length, choose(N + 1, 100), + failed_validation(F(?USERNAME, Val, regexp_that_requires_length_of_at_least(Length + 1))))). + +prop_regexp_passes_validation() -> + N = 5, + F = fun rabbit_credential_validator_password_regexp:validate/3, + ?FORALL(Val, binary(N), + passed_validation(F(?USERNAME, Val, regexp_that_requires_length_of_at_most(size(Val) + 1)))). + +%% +%% Helpers +%% + +passed_validation(ok) -> + true; +passed_validation({error, _}) -> + false. + +failed_validation(Result) -> + not passed_validation(Result). + +regexp_that_requires_length_of_at_least(N) when is_integer(N) -> + rabbit_misc:format("^[a-zA-Z0-9]{~p,~p}", [N, N + 10]). + +regexp_that_requires_length_of_at_most(N) when is_integer(N) -> + rabbit_misc:format("^[a-zA-Z0-9]{0,~p}", [N]). + +validator_backend(Config) -> + rabbit_ct_broker_helpers:rpc(Config, 0, rabbit_credential_validation, backend, []). diff --git a/deps/rabbit/test/unit_amqp091_content_framing_SUITE.erl b/deps/rabbit/test/unit_amqp091_content_framing_SUITE.erl new file mode 100644 index 0000000000..d483dbdd06 --- /dev/null +++ b/deps/rabbit/test/unit_amqp091_content_framing_SUITE.erl @@ -0,0 +1,231 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2011-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(unit_amqp091_content_framing_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("rabbit_common/include/rabbit.hrl"). +-include_lib("rabbit_common/include/rabbit_framing.hrl"). + +-compile(export_all). + +all() -> + [ + {group, parallel_tests} + ]. + +groups() -> + [ + {parallel_tests, [parallel], [ + write_table_with_invalid_existing_type, + invalid_existing_headers, + disparate_invalid_header_entries_accumulate_separately, + corrupt_or_invalid_headers_are_overwritten, + invalid_same_header_entry_accumulation, + content_framing, + content_transcoding, + table_codec + ]} + ]. + +%% ------------------------------------------------------------------- +%% Test Cases +%% ------------------------------------------------------------------- + +-define(XDEATH_TABLE, + [{<<"reason">>, longstr, <<"blah">>}, + {<<"queue">>, longstr, <<"foo.bar.baz">>}, + {<<"exchange">>, longstr, <<"my-exchange">>}, + {<<"routing-keys">>, array, []}]). + +-define(ROUTE_TABLE, [{<<"redelivered">>, bool, <<"true">>}]). + +-define(BAD_HEADER(K), {<<K>>, longstr, <<"bad ", K>>}). +-define(BAD_HEADER2(K, Suf), {<<K>>, longstr, <<"bad ", K, Suf>>}). +-define(FOUND_BAD_HEADER(K), {<<K>>, array, [{longstr, <<"bad ", K>>}]}). + +write_table_with_invalid_existing_type(_Config) -> + prepend_check(<<"header1">>, ?XDEATH_TABLE, [?BAD_HEADER("header1")]). + +invalid_existing_headers(_Config) -> + Headers = + prepend_check(<<"header2">>, ?ROUTE_TABLE, [?BAD_HEADER("header2")]), + {array, [{table, ?ROUTE_TABLE}]} = + rabbit_misc:table_lookup(Headers, <<"header2">>), + passed. + +disparate_invalid_header_entries_accumulate_separately(_Config) -> + BadHeaders = [?BAD_HEADER("header2")], + Headers = prepend_check(<<"header2">>, ?ROUTE_TABLE, BadHeaders), + Headers2 = prepend_check(<<"header1">>, ?XDEATH_TABLE, + [?BAD_HEADER("header1") | Headers]), + {table, [?FOUND_BAD_HEADER("header1"), + ?FOUND_BAD_HEADER("header2")]} = + rabbit_misc:table_lookup(Headers2, ?INVALID_HEADERS_KEY), + passed. + +corrupt_or_invalid_headers_are_overwritten(_Config) -> + Headers0 = [?BAD_HEADER("header1"), + ?BAD_HEADER("x-invalid-headers")], + Headers1 = prepend_check(<<"header1">>, ?XDEATH_TABLE, Headers0), + {table,[?FOUND_BAD_HEADER("header1"), + ?FOUND_BAD_HEADER("x-invalid-headers")]} = + rabbit_misc:table_lookup(Headers1, ?INVALID_HEADERS_KEY), + passed. + +invalid_same_header_entry_accumulation(_Config) -> + BadHeader1 = ?BAD_HEADER2("header1", "a"), + Headers = prepend_check(<<"header1">>, ?ROUTE_TABLE, [BadHeader1]), + Headers2 = prepend_check(<<"header1">>, ?ROUTE_TABLE, + [?BAD_HEADER2("header1", "b") | Headers]), + {table, InvalidHeaders} = + rabbit_misc:table_lookup(Headers2, ?INVALID_HEADERS_KEY), + {array, [{longstr,<<"bad header1b">>}, + {longstr,<<"bad header1a">>}]} = + rabbit_misc:table_lookup(InvalidHeaders, <<"header1">>), + passed. + +prepend_check(HeaderKey, HeaderTable, Headers) -> + Headers1 = rabbit_basic:prepend_table_header( + HeaderKey, HeaderTable, Headers), + {table, Invalid} = + rabbit_misc:table_lookup(Headers1, ?INVALID_HEADERS_KEY), + {Type, Value} = rabbit_misc:table_lookup(Headers, HeaderKey), + {array, [{Type, Value} | _]} = + rabbit_misc:table_lookup(Invalid, HeaderKey), + Headers1. + + +%% Test that content frames don't exceed frame-max +content_framing(_Config) -> + %% no content + passed = test_content_framing(4096, <<>>), + %% easily fit in one frame + passed = test_content_framing(4096, <<"Easy">>), + %% exactly one frame (empty frame = 8 bytes) + passed = test_content_framing(11, <<"One">>), + %% more than one frame + passed = test_content_framing(11, <<"More than one frame">>), + passed. + +test_content_framing(FrameMax, BodyBin) -> + [Header | Frames] = + rabbit_binary_generator:build_simple_content_frames( + 1, + rabbit_binary_generator:ensure_content_encoded( + rabbit_basic:build_content(#'P_basic'{}, BodyBin), + rabbit_framing_amqp_0_9_1), + FrameMax, + rabbit_framing_amqp_0_9_1), + %% header is formatted correctly and the size is the total of the + %% fragments + <<_FrameHeader:7/binary, _ClassAndWeight:4/binary, + BodySize:64/unsigned, _Rest/binary>> = list_to_binary(Header), + BodySize = size(BodyBin), + true = lists:all( + fun (ContentFrame) -> + FrameBinary = list_to_binary(ContentFrame), + %% assert + <<_TypeAndChannel:3/binary, + Size:32/unsigned, _Payload:Size/binary, 16#CE>> = + FrameBinary, + size(FrameBinary) =< FrameMax + end, Frames), + passed. + +content_transcoding(_Config) -> + %% there are no guarantees provided by 'clear' - it's just a hint + ClearDecoded = fun rabbit_binary_parser:clear_decoded_content/1, + ClearEncoded = fun rabbit_binary_generator:clear_encoded_content/1, + EnsureDecoded = + fun (C0) -> + C1 = rabbit_binary_parser:ensure_content_decoded(C0), + true = C1#content.properties =/= none, + C1 + end, + EnsureEncoded = + fun (Protocol) -> + fun (C0) -> + C1 = rabbit_binary_generator:ensure_content_encoded( + C0, Protocol), + true = C1#content.properties_bin =/= none, + C1 + end + end, + %% Beyond the assertions in Ensure*, the only testable guarantee + %% is that the operations should never fail. + %% + %% If we were using quickcheck we'd simply stuff all the above + %% into a generator for sequences of operations. In the absence of + %% quickcheck we pick particularly interesting sequences that: + %% + %% - execute every op twice since they are idempotent + %% - invoke clear_decoded, clear_encoded, decode and transcode + %% with one or both of decoded and encoded content present + [begin + sequence_with_content([Op]), + sequence_with_content([ClearEncoded, Op]), + sequence_with_content([ClearDecoded, Op]) + end || Op <- [ClearDecoded, ClearEncoded, EnsureDecoded, + EnsureEncoded(rabbit_framing_amqp_0_9_1), + EnsureEncoded(rabbit_framing_amqp_0_8)]], + passed. + +sequence_with_content(Sequence) -> + lists:foldl(fun (F, V) -> F(F(V)) end, + rabbit_binary_generator:ensure_content_encoded( + rabbit_basic:build_content(#'P_basic'{}, <<>>), + rabbit_framing_amqp_0_9_1), + Sequence). + +table_codec(_Config) -> + %% Note: this does not test inexact numbers (double and float) at the moment. + %% They won't pass the equality assertions. + Table = [{<<"longstr">>, longstr, <<"Here is a long string">>}, + {<<"signedint">>, signedint, 12345}, + {<<"decimal">>, decimal, {3, 123456}}, + {<<"timestamp">>, timestamp, 109876543209876}, + {<<"table">>, table, [{<<"one">>, signedint, 54321}, + {<<"two">>, longstr, + <<"A long string">>}]}, + {<<"byte">>, byte, -128}, + {<<"long">>, long, 1234567890}, + {<<"short">>, short, 655}, + {<<"bool">>, bool, true}, + {<<"binary">>, binary, <<"a binary string">>}, + {<<"unsignedbyte">>, unsignedbyte, 250}, + {<<"unsignedshort">>, unsignedshort, 65530}, + {<<"unsignedint">>, unsignedint, 4294967290}, + {<<"void">>, void, undefined}, + {<<"array">>, array, [{signedint, 54321}, + {longstr, <<"A long string">>}]} + ], + Binary = << + 7,"longstr", "S", 21:32, "Here is a long string", + 9,"signedint", "I", 12345:32/signed, + 7,"decimal", "D", 3, 123456:32, + 9,"timestamp", "T", 109876543209876:64, + 5,"table", "F", 31:32, % length of table + 3,"one", "I", 54321:32, + 3,"two", "S", 13:32, "A long string", + 4,"byte", "b", -128:8/signed, + 4,"long", "l", 1234567890:64, + 5,"short", "s", 655:16, + 4,"bool", "t", 1, + 6,"binary", "x", 15:32, "a binary string", + 12,"unsignedbyte", "B", 250:8/unsigned, + 13,"unsignedshort", "u", 65530:16/unsigned, + 11,"unsignedint", "i", 4294967290:32/unsigned, + 4,"void", "V", + 5,"array", "A", 23:32, + "I", 54321:32, + "S", 13:32, "A long string" + >>, + Binary = rabbit_binary_generator:generate_table(Table), + Table = rabbit_binary_parser:parse_table(Binary), + passed. diff --git a/deps/rabbit/test/unit_amqp091_server_properties_SUITE.erl b/deps/rabbit/test/unit_amqp091_server_properties_SUITE.erl new file mode 100644 index 0000000000..036fb8ce28 --- /dev/null +++ b/deps/rabbit/test/unit_amqp091_server_properties_SUITE.erl @@ -0,0 +1,144 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2011-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(unit_amqp091_server_properties_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("kernel/include/file.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-compile(export_all). + +-define(TIMEOUT_LIST_OPS_PASS, 5000). +-define(TIMEOUT, 30000). +-define(TIMEOUT_CHANNEL_EXCEPTION, 5000). + +-define(CLEANUP_QUEUE_NAME, <<"cleanup-queue">>). + +all() -> + [ + {group, parallel_tests} + ]. + +groups() -> + [ + {parallel_tests, [parallel], [ + configurable_server_properties + ]} + ]. + +suite() -> + [ + {timetrap, {minutes, 3}} + ]. + +%% ------------------------------------------------------------------- +%% Testsuite setup/teardown. +%% ------------------------------------------------------------------- + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(Group, Config) -> + case lists:member({group, Group}, all()) of + true -> + ClusterSize = 2, + Config1 = rabbit_ct_helpers:set_config(Config, [ + {rmq_nodename_suffix, Group}, + {rmq_nodes_count, ClusterSize} + ]), + rabbit_ct_helpers:run_steps(Config1, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()); + false -> + rabbit_ct_helpers:run_steps(Config, []) + end. + +end_per_group(Group, Config) -> + case lists:member({group, Group}, all()) of + true -> + rabbit_ct_helpers:run_steps(Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()); + false -> + Config + end. + +init_per_testcase(Testcase, Config) -> + Group = proplists:get_value(name, ?config(tc_group_properties, Config)), + Q = rabbit_data_coercion:to_binary(io_lib:format("~p_~p", [Group, Testcase])), + Config1 = rabbit_ct_helpers:set_config(Config, [{queue_name, Q}]), + rabbit_ct_helpers:testcase_started(Config1, Testcase). + +%% ------------------------------------------------------------------- +%% Test cases +%% ------------------------------------------------------------------- + +configurable_server_properties(Config) -> + passed = rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, configurable_server_properties1, [Config]). + +configurable_server_properties1(_Config) -> + %% List of the names of the built-in properties do we expect to find + BuiltInPropNames = [<<"product">>, <<"version">>, <<"platform">>, + <<"copyright">>, <<"information">>], + + Protocol = rabbit_framing_amqp_0_9_1, + + %% Verify that the built-in properties are initially present + ActualPropNames = [Key || {Key, longstr, _} <- + rabbit_reader:server_properties(Protocol)], + true = lists:all(fun (X) -> lists:member(X, ActualPropNames) end, + BuiltInPropNames), + + %% Get the initial server properties configured in the environment + {ok, ServerProperties} = application:get_env(rabbit, server_properties), + + %% Helper functions + ConsProp = fun (X) -> application:set_env(rabbit, + server_properties, + [X | ServerProperties]) end, + IsPropPresent = + fun (X) -> + lists:member(X, rabbit_reader:server_properties(Protocol)) + end, + + %% Add a wholly new property of the simplified {KeyAtom, StringValue} form + NewSimplifiedProperty = {NewHareKey, NewHareVal} = {hare, "soup"}, + ConsProp(NewSimplifiedProperty), + %% Do we find hare soup, appropriately formatted in the generated properties? + ExpectedHareImage = {list_to_binary(atom_to_list(NewHareKey)), + longstr, + list_to_binary(NewHareVal)}, + true = IsPropPresent(ExpectedHareImage), + + %% Add a wholly new property of the {BinaryKey, Type, Value} form + %% and check for it + NewProperty = {<<"new-bin-key">>, signedint, -1}, + ConsProp(NewProperty), + %% Do we find the new property? + true = IsPropPresent(NewProperty), + + %% Add a property that clobbers a built-in, and verify correct clobbering + {NewVerKey, NewVerVal} = NewVersion = {version, "X.Y.Z."}, + {BinNewVerKey, BinNewVerVal} = {list_to_binary(atom_to_list(NewVerKey)), + list_to_binary(NewVerVal)}, + ConsProp(NewVersion), + ClobberedServerProps = rabbit_reader:server_properties(Protocol), + %% Is the clobbering insert present? + true = IsPropPresent({BinNewVerKey, longstr, BinNewVerVal}), + %% Is the clobbering insert the only thing with the clobbering key? + [{BinNewVerKey, longstr, BinNewVerVal}] = + [E || {K, longstr, _V} = E <- ClobberedServerProps, K =:= BinNewVerKey], + + application:set_env(rabbit, server_properties, ServerProperties), + passed. diff --git a/deps/rabbit/test/unit_app_management_SUITE.erl b/deps/rabbit/test/unit_app_management_SUITE.erl new file mode 100644 index 0000000000..e08f151d57 --- /dev/null +++ b/deps/rabbit/test/unit_app_management_SUITE.erl @@ -0,0 +1,105 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2011-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(unit_app_management_SUITE). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("kernel/include/file.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). + +-compile(export_all). + +-define(TIMEOUT, 30000). + +all() -> + [ + {group, non_parallel_tests} + ]. + +groups() -> + [ + {non_parallel_tests, [], [ + app_management + ]} + ]. + +%% ------------------------------------------------------------------- +%% Testsuite setup/teardown. +%% ------------------------------------------------------------------- + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(Group, Config) -> + Config1 = rabbit_ct_helpers:set_config(Config, [ + {rmq_nodename_suffix, Group}, + {rmq_nodes_count, 2} + ]), + rabbit_ct_helpers:run_steps(Config1, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()). + +end_per_group(_Group, Config) -> + rabbit_ct_helpers:run_steps(Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()). + +init_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_started(Config, Testcase). + +end_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_finished(Config, Testcase). + +%% ------------------------------------------------------------------- +%% Application management. +%% ------------------------------------------------------------------- + +app_management(Config) -> + passed = rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, app_management1, [Config]). + +app_management1(_Config) -> + wait_for_application(rabbit), + %% Starting, stopping and diagnostics. Note that we don't try + %% 'report' when the rabbit app is stopped and that we enable + %% tracing for the duration of this function. + ok = rabbit_trace:start(<<"/">>), + ok = rabbit:stop(), + ok = rabbit:stop(), + ok = no_exceptions(rabbit, status, []), + ok = no_exceptions(rabbit, environment, []), + ok = rabbit:start(), + ok = rabbit:start(), + ok = no_exceptions(rabbit, status, []), + ok = no_exceptions(rabbit, environment, []), + ok = rabbit_trace:stop(<<"/">>), + passed. + +no_exceptions(Mod, Fun, Args) -> + try erlang:apply(Mod, Fun, Args) of _ -> ok + catch Type:Ex -> {Type, Ex} + end. + +wait_for_application(Application) -> + wait_for_application(Application, 5000). + +wait_for_application(_, Time) when Time =< 0 -> + {error, timeout}; +wait_for_application(Application, Time) -> + Interval = 100, + case lists:keyfind(Application, 1, application:which_applications()) of + false -> + timer:sleep(Interval), + wait_for_application(Application, Time - Interval); + _ -> ok + end. diff --git a/deps/rabbit/test/unit_cluster_formation_locking_mocks_SUITE.erl b/deps/rabbit/test/unit_cluster_formation_locking_mocks_SUITE.erl new file mode 100644 index 0000000000..41dd685694 --- /dev/null +++ b/deps/rabbit/test/unit_cluster_formation_locking_mocks_SUITE.erl @@ -0,0 +1,71 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. +%% +-module(unit_cluster_formation_locking_mocks_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-compile(export_all). + +all() -> + [ + {group, non_parallel_tests} + ]. + +groups() -> + [ + {non_parallel_tests, [], [ + init_with_lock_exits_after_errors, + init_with_lock_ignore_after_errors, + init_with_lock_not_supported, + init_with_lock_supported + ]} + ]. + +init_per_testcase(Testcase, Config) when Testcase == init_with_lock_exits_after_errors; + Testcase == init_with_lock_not_supported; + Testcase == init_with_lock_supported -> + application:set_env(rabbit, cluster_formation, + [{peer_discover_backend, peer_discover_classic_config}, + {lock_acquisition_failure_mode, fail}]), + ok = meck:new(rabbit_peer_discovery_classic_config, [passthrough]), + Config; +init_per_testcase(init_with_lock_ignore_after_errors, Config) -> + application:set_env(rabbit, cluster_formation, + [{peer_discover_backend, peer_discover_classic_config}, + {lock_acquisition_failure_mode, ignore}]), + ok = meck:new(rabbit_peer_discovery_classic_config, [passthrough]), + Config. + +end_per_testcase(_, _) -> + meck:unload(), + application:unset_env(rabbit, cluster_formation). + +init_with_lock_exits_after_errors(_Config) -> + meck:expect(rabbit_peer_discovery_classic_config, lock, fun(_) -> {error, "test error"} end), + ?assertExit(cannot_acquire_startup_lock, rabbit_mnesia:init_with_lock(2, 10, fun() -> ok end)), + ?assert(meck:validate(rabbit_peer_discovery_classic_config)), + passed. + +init_with_lock_ignore_after_errors(_Config) -> + meck:expect(rabbit_peer_discovery_classic_config, lock, fun(_) -> {error, "test error"} end), + ?assertEqual(ok, rabbit_mnesia:init_with_lock(2, 10, fun() -> ok end)), + ?assert(meck:validate(rabbit_peer_discovery_classic_config)), + passed. + +init_with_lock_not_supported(_Config) -> + meck:expect(rabbit_peer_discovery_classic_config, lock, fun(_) -> not_supported end), + ?assertEqual(ok, rabbit_mnesia:init_with_lock(2, 10, fun() -> ok end)), + ?assert(meck:validate(rabbit_peer_discovery_classic_config)), + passed. + +init_with_lock_supported(_Config) -> + meck:expect(rabbit_peer_discovery_classic_config, lock, fun(_) -> {ok, data} end), + meck:expect(rabbit_peer_discovery_classic_config, unlock, fun(data) -> ok end), + ?assertEqual(ok, rabbit_mnesia:init_with_lock(2, 10, fun() -> ok end)), + ?assert(meck:validate(rabbit_peer_discovery_classic_config)), + passed. diff --git a/deps/rabbit/test/unit_collections_SUITE.erl b/deps/rabbit/test/unit_collections_SUITE.erl new file mode 100644 index 0000000000..1cbf65efce --- /dev/null +++ b/deps/rabbit/test/unit_collections_SUITE.erl @@ -0,0 +1,51 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2011-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(unit_collections_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-compile(export_all). + +all() -> + [ + {group, parallel_tests} + ]. + +groups() -> + [ + {parallel_tests, [parallel], [ + pmerge, + plmerge, + unfold + ]} + ]. + +%% ------------------------------------------------------------------- +%% Test Cases +%% ------------------------------------------------------------------- + +pmerge(_Config) -> + P = [{a, 1}, {b, 2}], + P = rabbit_misc:pmerge(a, 3, P), + [{c, 3} | P] = rabbit_misc:pmerge(c, 3, P), + passed. + +plmerge(_Config) -> + P1 = [{a, 1}, {b, 2}, {c, 3}], + P2 = [{a, 2}, {d, 4}], + [{a, 1}, {b, 2}, {c, 3}, {d, 4}] = rabbit_misc:plmerge(P1, P2), + passed. + +unfold(_Config) -> + {[], test} = rabbit_misc:unfold(fun (_V) -> false end, test), + List = lists:seq(2,20,2), + {List, 0} = rabbit_misc:unfold(fun (0) -> false; + (N) -> {true, N*2, N-1} + end, 10), + passed. diff --git a/deps/rabbit/test/unit_config_value_encryption_SUITE.erl b/deps/rabbit/test/unit_config_value_encryption_SUITE.erl new file mode 100644 index 0000000000..7536005797 --- /dev/null +++ b/deps/rabbit/test/unit_config_value_encryption_SUITE.erl @@ -0,0 +1,233 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2011-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(unit_config_value_encryption_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-compile(export_all). + +all() -> + [ + {group, sequential_tests} + ]. + +groups() -> + [ + {sequential_tests, [], [ + decrypt_start_app, + decrypt_start_app_file, + decrypt_start_app_undefined, + decrypt_start_app_wrong_passphrase, + decrypt_config, + rabbitmqctl_encode + ]} + ]. + +init_per_testcase(TC, Config) when TC =:= decrypt_start_app; + TC =:= decrypt_start_app_file; + TC =:= decrypt_start_app_undefined; + TC =:= decrypt_start_app_wrong_passphrase -> + application:set_env(rabbit, feature_flags_file, "", [{persistent, true}]), + Config; +init_per_testcase(_Testcase, Config) -> + Config. + +end_per_testcase(_TC, _Config) -> + ok. + +%% ------------------------------------------------------------------- +%% Test Cases +%% ------------------------------------------------------------------- + +decrypt_config(_Config) -> + %% Take all available block ciphers. + Hashes = rabbit_pbe:supported_hashes(), + Ciphers = rabbit_pbe:supported_ciphers(), + Iterations = [1, 10, 100, 1000], + %% Loop through all hashes, ciphers and iterations. + _ = [begin + PassPhrase = crypto:strong_rand_bytes(16), + do_decrypt_config({C, H, I, PassPhrase}) + end || H <- Hashes, C <- Ciphers, I <- Iterations], + ok. + +do_decrypt_config(Algo = {C, H, I, P}) -> + ok = application:load(rabbit), + RabbitConfig = application:get_all_env(rabbit), + %% Encrypt a few values in configuration. + %% Common cases. + _ = [encrypt_value(Key, Algo) || Key <- [ + tcp_listeners, + num_tcp_acceptors, + ssl_options, + vm_memory_high_watermark, + default_pass, + default_permissions, + cluster_nodes, + auth_mechanisms, + msg_store_credit_disc_bound]], + %% Special case: encrypt a value in a list. + {ok, [LoopbackUser]} = application:get_env(rabbit, loopback_users), + {encrypted, EncLoopbackUser} = rabbit_pbe:encrypt_term(C, H, I, P, LoopbackUser), + application:set_env(rabbit, loopback_users, [{encrypted, EncLoopbackUser}]), + %% Special case: encrypt a value in a key/value list. + {ok, TCPOpts} = application:get_env(rabbit, tcp_listen_options), + {_, Backlog} = lists:keyfind(backlog, 1, TCPOpts), + {_, Linger} = lists:keyfind(linger, 1, TCPOpts), + {encrypted, EncBacklog} = rabbit_pbe:encrypt_term(C, H, I, P, Backlog), + {encrypted, EncLinger} = rabbit_pbe:encrypt_term(C, H, I, P, Linger), + TCPOpts1 = lists:keyreplace(backlog, 1, TCPOpts, {backlog, {encrypted, EncBacklog}}), + TCPOpts2 = lists:keyreplace(linger, 1, TCPOpts1, {linger, {encrypted, EncLinger}}), + application:set_env(rabbit, tcp_listen_options, TCPOpts2), + %% Decrypt configuration. + rabbit_prelaunch_conf:decrypt_config([rabbit], Algo), + %% Check that configuration was decrypted properly. + RabbitConfig = application:get_all_env(rabbit), + ok = application:unload(rabbit), + ok. + +encrypt_value(Key, {C, H, I, P}) -> + {ok, Value} = application:get_env(rabbit, Key), + {encrypted, EncValue} = rabbit_pbe:encrypt_term(C, H, I, P, Value), + application:set_env(rabbit, Key, {encrypted, EncValue}). + +decrypt_start_app(Config) -> + do_decrypt_start_app(Config, "hello"). + +decrypt_start_app_file(Config) -> + do_decrypt_start_app(Config, {file, ?config(data_dir, Config) ++ "/rabbit_shovel_test.passphrase"}). + +do_decrypt_start_app(Config, Passphrase) -> + %% Configure rabbit for decrypting configuration. + application:set_env(rabbit, config_entry_decoder, [ + {cipher, aes_cbc256}, + {hash, sha512}, + {iterations, 1000}, + {passphrase, Passphrase} + ], [{persistent, true}]), + %% Add the path to our test application. + code:add_path(?config(data_dir, Config) ++ "/lib/rabbit_shovel_test/ebin"), + %% Attempt to start our test application. + %% + %% We expect a failure *after* the decrypting has been done. + try + rabbit:start_apps([rabbit_shovel_test], #{rabbit => temporary}) + catch _:_ -> + ok + end, + %% Check if the values have been decrypted. + {ok, Shovels} = application:get_env(rabbit_shovel_test, shovels), + {_, FirstShovel} = lists:keyfind(my_first_shovel, 1, Shovels), + {_, Sources} = lists:keyfind(sources, 1, FirstShovel), + {_, Brokers} = lists:keyfind(brokers, 1, Sources), + ["amqp://fred:secret@host1.domain/my_vhost", + "amqp://john:secret@host2.domain/my_vhost"] = Brokers, + ok. + +decrypt_start_app_undefined(Config) -> + %% Configure rabbit for decrypting configuration. + application:set_env(rabbit, config_entry_decoder, [ + {cipher, aes_cbc256}, + {hash, sha512}, + {iterations, 1000} + %% No passphrase option! + ], [{persistent, true}]), + %% Add the path to our test application. + code:add_path(?config(data_dir, Config) ++ "/lib/rabbit_shovel_test/ebin"), + %% Attempt to start our test application. + %% + %% We expect a failure during decryption because the passphrase is missing. + try + rabbit:start_apps([rabbit_shovel_test], #{rabbit => temporary}) + catch + throw:{bad_config_entry_decoder, missing_passphrase} -> ok; + _:Exception -> exit({unexpected_exception, Exception}) + end. + +decrypt_start_app_wrong_passphrase(Config) -> + %% Configure rabbit for decrypting configuration. + application:set_env(rabbit, config_entry_decoder, [ + {cipher, aes_cbc256}, + {hash, sha512}, + {iterations, 1000}, + {passphrase, "wrong passphrase"} + ], [{persistent, true}]), + %% Add the path to our test application. + code:add_path(?config(data_dir, Config) ++ "/lib/rabbit_shovel_test/ebin"), + %% Attempt to start our test application. + %% + %% We expect a failure during decryption because the passphrase is wrong. + try + rabbit:start_apps([rabbit_shovel_test], #{rabbit => temporary}) + catch + throw:{config_decryption_error, _, _} -> ok; + _:Exception -> exit({unexpected_exception, Exception}) + end. + +rabbitmqctl_encode(_Config) -> + % list ciphers and hashes + {ok, _} = rabbit_control_pbe:list_ciphers(), + {ok, _} = rabbit_control_pbe:list_hashes(), + % incorrect ciphers, hashes and iteration number + {error, _} = rabbit_control_pbe:encode(funny_cipher, undefined, undefined, undefined), + {error, _} = rabbit_control_pbe:encode(undefined, funny_hash, undefined, undefined), + {error, _} = rabbit_control_pbe:encode(undefined, undefined, -1, undefined), + {error, _} = rabbit_control_pbe:encode(undefined, undefined, 0, undefined), + % incorrect number of arguments + {error, _} = rabbit_control_pbe:encode( + rabbit_pbe:default_cipher(), rabbit_pbe:default_hash(), rabbit_pbe:default_iterations(), + [] + ), + {error, _} = rabbit_control_pbe:encode( + rabbit_pbe:default_cipher(), rabbit_pbe:default_hash(), rabbit_pbe:default_iterations(), + [undefined] + ), + {error, _} = rabbit_control_pbe:encode( + rabbit_pbe:default_cipher(), rabbit_pbe:default_hash(), rabbit_pbe:default_iterations(), + [undefined, undefined, undefined] + ), + + % encrypt/decrypt + % string + rabbitmqctl_encode_encrypt_decrypt("foobar"), + % binary + rabbitmqctl_encode_encrypt_decrypt("<<\"foobar\">>"), + % tuple + rabbitmqctl_encode_encrypt_decrypt("{password,<<\"secret\">>}"), + + ok. + +rabbitmqctl_encode_encrypt_decrypt(Secret) -> + PassPhrase = "passphrase", + {ok, Output} = rabbit_control_pbe:encode( + rabbit_pbe:default_cipher(), rabbit_pbe:default_hash(), rabbit_pbe:default_iterations(), + [Secret, PassPhrase] + ), + {encrypted, Encrypted} = rabbit_control_pbe:evaluate_input_as_term(lists:flatten(Output)), + + {ok, Result} = rabbit_control_pbe:decode( + rabbit_pbe:default_cipher(), rabbit_pbe:default_hash(), rabbit_pbe:default_iterations(), + [lists:flatten(io_lib:format("~p", [Encrypted])), PassPhrase] + ), + Secret = lists:flatten(Result), + % decrypt with {encrypted, ...} form as input + {ok, Result} = rabbit_control_pbe:decode( + rabbit_pbe:default_cipher(), rabbit_pbe:default_hash(), rabbit_pbe:default_iterations(), + [lists:flatten(io_lib:format("~p", [{encrypted, Encrypted}])), PassPhrase] + ), + + % wrong passphrase + {error, _} = rabbit_control_pbe:decode( + rabbit_pbe:default_cipher(), rabbit_pbe:default_hash(), rabbit_pbe:default_iterations(), + [lists:flatten(io_lib:format("~p", [Encrypted])), PassPhrase ++ " "] + ), + {error, _} = rabbit_control_pbe:decode( + rabbit_pbe:default_cipher(), rabbit_pbe:default_hash(), rabbit_pbe:default_iterations(), + [lists:flatten(io_lib:format("~p", [{encrypted, Encrypted}])), PassPhrase ++ " "] + ). diff --git a/deps/rabbit/test/unit_config_value_encryption_SUITE_data/lib/rabbit_shovel_test/ebin/rabbit_shovel_test.app b/deps/rabbit/test/unit_config_value_encryption_SUITE_data/lib/rabbit_shovel_test/ebin/rabbit_shovel_test.app new file mode 100644 index 0000000000..a8481c9aa4 --- /dev/null +++ b/deps/rabbit/test/unit_config_value_encryption_SUITE_data/lib/rabbit_shovel_test/ebin/rabbit_shovel_test.app @@ -0,0 +1,46 @@ +{application, rabbit_shovel_test, + [{description, "Test .app file for tests for encrypting configuration"}, + {vsn, ""}, + {modules, []}, + {env, [ {shovels, [ {my_first_shovel, + [ {sources, + [ {brokers, [ {encrypted, <<"CfJXuka/uJYsqAtiJnwKpSY4moMPcOBh4sO8XDcdmhXbVYGKCDLKEilWPMfvOAQ2lN1BQneGn6bvDZi2+gDu6iHVKfafQAZSv8zcsVB3uYdBXFzqTCWO8TAsgG6LUMPT">>} + , {encrypted, <<"dBO6n+G1OiBwZeLXhvmNYeTE57nhBOmicUBF34zo4nQjerzQaNoEk8GA2Ts5PzMhYeO6U6Y9eEmheqIr9Gzh2duLZic65ZMQtIKNpWcZJllEhGpk7aV1COr23Yur9fWG">>} + ]} + , {declarations, [ {'exchange.declare', + [ {exchange, <<"my_fanout">>} + , {type, <<"fanout">>} + , durable + ]} + , {'queue.declare', + [{arguments, + [{<<"x-message-ttl">>, long, 60000}]}]} + , {'queue.bind', + [ {exchange, <<"my_direct">>} + , {queue, <<>>} + ]} + ]} + ]} + , {destinations, + [ {broker, "amqp://"} + , {declarations, [ {'exchange.declare', + [ {exchange, <<"my_direct">>} + , {type, <<"direct">>} + , durable + ]} + ]} + ]} + , {queue, <<>>} + , {prefetch_count, 10} + , {ack_mode, on_confirm} + , {publish_properties, [ {delivery_mode, 2} ]} + , {add_forward_headers, true} + , {publish_fields, [ {exchange, <<"my_direct">>} + , {routing_key, <<"from_shovel">>} + ]} + , {reconnect_delay, 5} + ]} + ]} + ]}, + + {applications, [kernel, stdlib]}]}. diff --git a/deps/rabbit/test/unit_config_value_encryption_SUITE_data/rabbit_shovel_test.passphrase b/deps/rabbit/test/unit_config_value_encryption_SUITE_data/rabbit_shovel_test.passphrase new file mode 100644 index 0000000000..ce01362503 --- /dev/null +++ b/deps/rabbit/test/unit_config_value_encryption_SUITE_data/rabbit_shovel_test.passphrase @@ -0,0 +1 @@ +hello diff --git a/deps/rabbit/test/unit_connection_tracking_SUITE.erl b/deps/rabbit/test/unit_connection_tracking_SUITE.erl new file mode 100644 index 0000000000..4ea1744fa7 --- /dev/null +++ b/deps/rabbit/test/unit_connection_tracking_SUITE.erl @@ -0,0 +1,119 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2011-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(unit_connection_tracking_SUITE). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("kernel/include/file.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). + +-compile(export_all). + +-define(TIMEOUT, 30000). + +all() -> + [ + {group, non_parallel_tests} + ]. + +groups() -> + [ + {non_parallel_tests, [], [ + exchange_count, + queue_count, + connection_count, + connection_lookup + ]} + ]. + +%% ------------------------------------------------------------------- +%% Test suite setup/teardown. +%% ------------------------------------------------------------------- + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(Group, Config) -> + Config1 = rabbit_ct_helpers:set_config(Config, [ + {rmq_nodename_suffix, Group}, + {rmq_nodes_count, 1} + ]), + rabbit_ct_helpers:run_steps(Config1, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()). + +end_per_group(_Group, Config) -> + rabbit_ct_helpers:run_steps(Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()). + +init_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_started(Config, Testcase). + +end_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_finished(Config, Testcase). + + +%% --------------------------------------------------------------------------- +%% Count functions for management only API purposes +%% --------------------------------------------------------------------------- + +exchange_count(Config) -> + %% Default exchanges == 7 + ?assertEqual(7, rabbit_ct_broker_helpers:rpc(Config, 0, rabbit_exchange, count, [])). + +queue_count(Config) -> + Conn = rabbit_ct_client_helpers:open_connection(Config, 0), + {ok, Ch} = amqp_connection:open_channel(Conn), + amqp_channel:call(Ch, #'queue.declare'{ queue = <<"my-queue">> }), + + ?assertEqual(1, rabbit_ct_broker_helpers:rpc(Config, 0, rabbit_amqqueue, count, [])), + + amqp_channel:call(Ch, #'queue.delete'{ queue = <<"my-queue">> }), + rabbit_ct_client_helpers:close_channel(Ch), + rabbit_ct_client_helpers:close_connection(Conn), + ok. + +%% connection_count/1 has been failing on Travis. This seems a legit failure, as the registering +%% of connections in the tracker is async. `rabbit_connection_tracking_handler` receives a rabbit +%% event with `connection_created`, which then forwards as a cast to `rabbit_connection_tracker` +%% for register. We should wait a reasonable amount of time for the counter to increase before +%% failing. +connection_count(Config) -> + Conn = rabbit_ct_client_helpers:open_connection(Config, 0), + + rabbit_ct_helpers:await_condition( + fun() -> + rabbit_ct_broker_helpers:rpc(Config, 0, rabbit_connection_tracking, count, []) == 1 + end, 30000), + + rabbit_ct_client_helpers:close_connection(Conn), + ok. + +connection_lookup(Config) -> + Conn = rabbit_ct_client_helpers:open_connection(Config, 0), + + %% Let's wait until the connection is registered, otherwise this test could fail in a slow + %% machine as connection tracking is asynchronous + rabbit_ct_helpers:await_condition( + fun() -> + rabbit_ct_broker_helpers:rpc(Config, 0, rabbit_connection_tracking, count, []) == 1 + end, 30000), + + [Connection] = rabbit_ct_broker_helpers:rpc(Config, 0, rabbit_connection_tracking, list, []), + ?assertMatch(Connection, rabbit_ct_broker_helpers:rpc(Config, 0, rabbit_connection_tracking, + lookup, + [Connection#tracked_connection.name])), + + rabbit_ct_client_helpers:close_connection(Conn), + ok. diff --git a/deps/rabbit/test/unit_credit_flow_SUITE.erl b/deps/rabbit/test/unit_credit_flow_SUITE.erl new file mode 100644 index 0000000000..ffad444dde --- /dev/null +++ b/deps/rabbit/test/unit_credit_flow_SUITE.erl @@ -0,0 +1,90 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2011-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(unit_credit_flow_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-compile(export_all). + +all() -> + [ + {group, sequential_tests} + ]. + +groups() -> + [ + {sequential_tests, [], [ + credit_flow_settings + ]} + ]. + +%% ------------------------------------------------------------------- +%% Testsuite setup/teardown +%% ------------------------------------------------------------------- + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(Group, Config) -> + Config1 = rabbit_ct_helpers:set_config(Config, [ + {rmq_nodename_suffix, Group}, + {rmq_nodes_count, 1} + ]), + rabbit_ct_helpers:run_steps(Config1, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()). + +end_per_group(_Group, Config) -> + rabbit_ct_helpers:run_steps(Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()). + +init_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_started(Config, Testcase). + +end_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_finished(Config, Testcase). + + +%% --------------------------------------------------------------------------- +%% Test Cases +%% --------------------------------------------------------------------------- + +credit_flow_settings(Config) -> + rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, credit_flow_settings1, [Config]). + +credit_flow_settings1(_Config) -> + passed = test_proc(400, 200, {400, 200}), + passed = test_proc(600, 300), + passed. + +test_proc(InitialCredit, MoreCreditAfter) -> + test_proc(InitialCredit, MoreCreditAfter, {InitialCredit, MoreCreditAfter}). +test_proc(InitialCredit, MoreCreditAfter, Settings) -> + Pid = spawn(?MODULE, dummy, [Settings]), + Pid ! {credit, self()}, + {InitialCredit, MoreCreditAfter} = + receive + {credit, Val} -> Val + end, + passed. + +dummy(Settings) -> + credit_flow:send(self()), + receive + {credit, From} -> + From ! {credit, Settings}; + _ -> + dummy(Settings) + end. diff --git a/deps/rabbit/test/unit_disk_monitor_SUITE.erl b/deps/rabbit/test/unit_disk_monitor_SUITE.erl new file mode 100644 index 0000000000..bc21114c12 --- /dev/null +++ b/deps/rabbit/test/unit_disk_monitor_SUITE.erl @@ -0,0 +1,90 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2011-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(unit_disk_monitor_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-compile(export_all). + +-define(TIMEOUT, 30000). + +all() -> + [ + {group, sequential_tests} + ]. + +groups() -> + [ + {sequential_tests, [], [ + set_disk_free_limit_command + ]} + ]. + +%% ------------------------------------------------------------------- +%% Testsuite setup/teardown +%% ------------------------------------------------------------------- + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(Group, Config) -> + Config1 = rabbit_ct_helpers:set_config(Config, [ + {rmq_nodename_suffix, Group}, + {rmq_nodes_count, 1} + ]), + rabbit_ct_helpers:run_steps(Config1, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()). + +end_per_group(_Group, Config) -> + rabbit_ct_helpers:run_steps(Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()). + +init_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_started(Config, Testcase). + +end_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_finished(Config, Testcase). + + +%% ------------------------------------------------------------------- +%% Test cases +%% ------------------------------------------------------------------- + +set_disk_free_limit_command(Config) -> + passed = rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, set_disk_free_limit_command1, [Config]). + +set_disk_free_limit_command1(_Config) -> + %% Use an integer + rabbit_disk_monitor:set_disk_free_limit({mem_relative, 1}), + disk_free_limit_to_total_memory_ratio_is(1), + + %% Use a float + rabbit_disk_monitor:set_disk_free_limit({mem_relative, 1.5}), + disk_free_limit_to_total_memory_ratio_is(1.5), + + %% use an absolute value + rabbit_disk_monitor:set_disk_free_limit("70MiB"), + ?assertEqual(73400320, rabbit_disk_monitor:get_disk_free_limit()), + + rabbit_disk_monitor:set_disk_free_limit("50MB"), + ?assertEqual(50 * 1000 * 1000, rabbit_disk_monitor:get_disk_free_limit()), + passed. + +disk_free_limit_to_total_memory_ratio_is(MemRatio) -> + ExpectedLimit = MemRatio * vm_memory_monitor:get_total_memory(), + % Total memory is unstable, so checking order + true = ExpectedLimit/rabbit_disk_monitor:get_disk_free_limit() < 1.2, + true = ExpectedLimit/rabbit_disk_monitor:get_disk_free_limit() > 0.98. diff --git a/deps/rabbit/test/unit_disk_monitor_mocks_SUITE.erl b/deps/rabbit/test/unit_disk_monitor_mocks_SUITE.erl new file mode 100644 index 0000000000..af78d0d134 --- /dev/null +++ b/deps/rabbit/test/unit_disk_monitor_mocks_SUITE.erl @@ -0,0 +1,112 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2011-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(unit_disk_monitor_mocks_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-compile(export_all). + +-define(TIMEOUT, 30000). + +all() -> + [ + {group, sequential_tests} + ]. + +groups() -> + [ + {sequential_tests, [], [ + disk_monitor, + disk_monitor_enable + ]} + ]. + +%% ------------------------------------------------------------------- +%% Testsuite setup/teardown +%% ------------------------------------------------------------------- + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(Group, Config) -> + Config1 = rabbit_ct_helpers:set_config(Config, [ + {rmq_nodename_suffix, Group}, + {rmq_nodes_count, 1} + ]), + rabbit_ct_helpers:run_steps(Config1, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()). + +end_per_group(_Group, Config) -> + rabbit_ct_helpers:run_steps(Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()). + +init_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_started(Config, Testcase). + +end_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_finished(Config, Testcase). + +%% ------------------------------------------------------------------- +%% Test cases +%% ------------------------------------------------------------------- + +disk_monitor(Config) -> + passed = rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, disk_monitor1, [Config]). + +disk_monitor1(_Config) -> + %% Issue: rabbitmq-server #91 + %% os module could be mocked using 'unstick', however it may have undesired + %% side effects in following tests. Thus, we mock at rabbit_misc level + ok = meck:new(rabbit_misc, [passthrough]), + ok = meck:expect(rabbit_misc, os_cmd, fun(_) -> "\n" end), + ok = rabbit_sup:stop_child(rabbit_disk_monitor_sup), + ok = rabbit_sup:start_delayed_restartable_child(rabbit_disk_monitor, [1000]), + meck:unload(rabbit_misc), + passed. + +disk_monitor_enable(Config) -> + passed = rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, disk_monitor_enable1, [Config]). + +disk_monitor_enable1(_Config) -> + ok = meck:new(rabbit_misc, [passthrough]), + ok = meck:expect(rabbit_misc, os_cmd, fun(_) -> "\n" end), + application:set_env(rabbit, disk_monitor_failure_retries, 20000), + application:set_env(rabbit, disk_monitor_failure_retry_interval, 100), + ok = rabbit_sup:stop_child(rabbit_disk_monitor_sup), + ok = rabbit_sup:start_delayed_restartable_child(rabbit_disk_monitor, [1000]), + undefined = rabbit_disk_monitor:get_disk_free(), + Cmd = case os:type() of + {win32, _} -> " Le volume dans le lecteur C n’a pas de nom.\n" + " Le numéro de série du volume est 707D-5BDC\n" + "\n" + " Répertoire de C:\Users\n" + "\n" + "10/12/2015 11:01 <DIR> .\n" + "10/12/2015 11:01 <DIR> ..\n" + " 0 fichier(s) 0 octets\n" + " 2 Rép(s) 758537121792 octets libres\n"; + _ -> "Filesystem 1024-blocks Used Available Capacity iused ifree %iused Mounted on\n" + "/dev/disk1 975798272 234783364 740758908 25% 58759839 185189727 24% /\n" + end, + ok = meck:expect(rabbit_misc, os_cmd, fun(_) -> Cmd end), + timer:sleep(1000), + Bytes = 740758908 * 1024, + Bytes = rabbit_disk_monitor:get_disk_free(), + meck:unload(rabbit_misc), + application:set_env(rabbit, disk_monitor_failure_retries, 10), + application:set_env(rabbit, disk_monitor_failure_retry_interval, 120000), + passed. diff --git a/deps/rabbit/test/unit_file_handle_cache_SUITE.erl b/deps/rabbit/test/unit_file_handle_cache_SUITE.erl new file mode 100644 index 0000000000..f2252aa2b5 --- /dev/null +++ b/deps/rabbit/test/unit_file_handle_cache_SUITE.erl @@ -0,0 +1,278 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2011-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(unit_file_handle_cache_SUITE). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("kernel/include/file.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). + +-compile(export_all). + +-define(TIMEOUT, 30000). + +all() -> + [ + {group, non_parallel_tests} + ]. + +groups() -> + [ + {non_parallel_tests, [], [ + file_handle_cache, %% Change FHC limit. + file_handle_cache_reserve, + file_handle_cache_reserve_release, + file_handle_cache_reserve_above_limit, + file_handle_cache_reserve_monitor, + file_handle_cache_reserve_open_file_above_limit + ]} + ]. + +%% ------------------------------------------------------------------- +%% Testsuite setup/teardown. +%% ------------------------------------------------------------------- + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(Group, Config) -> + Config1 = rabbit_ct_helpers:set_config(Config, [ + {rmq_nodename_suffix, Group}, + {rmq_nodes_count, 2} + ]), + rabbit_ct_helpers:run_steps(Config1, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps() ++ [ + fun setup_file_handle_cache/1 + ]). + +setup_file_handle_cache(Config) -> + ok = rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, setup_file_handle_cache1, []), + Config. + +setup_file_handle_cache1() -> + %% FIXME: Why are we doing this? + application:set_env(rabbit, file_handles_high_watermark, 10), + ok = file_handle_cache:set_limit(10), + ok. + +end_per_group(_Group, Config) -> + rabbit_ct_helpers:run_steps(Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()). + +init_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_started(Config, Testcase). + +end_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_finished(Config, Testcase). + + +%% --------------------------------------------------------------------------- +%% file_handle_cache. +%% --------------------------------------------------------------------------- + +file_handle_cache(Config) -> + passed = rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, file_handle_cache1, [Config]). + +file_handle_cache1(_Config) -> + %% test copying when there is just one spare handle + Limit = file_handle_cache:get_limit(), + ok = file_handle_cache:set_limit(5), %% 1 or 2 sockets, 2 msg_stores + TmpDir = filename:join(rabbit_mnesia:dir(), "tmp"), + ok = filelib:ensure_dir(filename:join(TmpDir, "nothing")), + [Src1, Dst1, Src2, Dst2] = Files = + [filename:join(TmpDir, Str) || Str <- ["file1", "file2", "file3", "file4"]], + Content = <<"foo">>, + CopyFun = fun (Src, Dst) -> + {ok, Hdl} = prim_file:open(Src, [binary, write]), + ok = prim_file:write(Hdl, Content), + ok = prim_file:sync(Hdl), + prim_file:close(Hdl), + + {ok, SrcHdl} = file_handle_cache:open(Src, [read], []), + {ok, DstHdl} = file_handle_cache:open(Dst, [write], []), + Size = size(Content), + {ok, Size} = file_handle_cache:copy(SrcHdl, DstHdl, Size), + ok = file_handle_cache:delete(SrcHdl), + ok = file_handle_cache:delete(DstHdl) + end, + Pid = spawn(fun () -> {ok, Hdl} = file_handle_cache:open( + filename:join(TmpDir, "file5"), + [write], []), + receive {next, Pid1} -> Pid1 ! {next, self()} end, + file_handle_cache:delete(Hdl), + %% This will block and never return, so we + %% exercise the fhc tidying up the pending + %% queue on the death of a process. + ok = CopyFun(Src1, Dst1) + end), + ok = CopyFun(Src1, Dst1), + ok = file_handle_cache:set_limit(2), + Pid ! {next, self()}, + receive {next, Pid} -> ok end, + timer:sleep(100), + Pid1 = spawn(fun () -> CopyFun(Src2, Dst2) end), + timer:sleep(100), + erlang:monitor(process, Pid), + erlang:monitor(process, Pid1), + exit(Pid, kill), + exit(Pid1, kill), + receive {'DOWN', _MRef, process, Pid, _Reason} -> ok end, + receive {'DOWN', _MRef1, process, Pid1, _Reason1} -> ok end, + [file:delete(File) || File <- Files], + ok = file_handle_cache:set_limit(Limit), + passed. + +file_handle_cache_reserve(Config) -> + passed = rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, file_handle_cache_reserve1, [Config]). + +file_handle_cache_reserve1(_Config) -> + Limit = file_handle_cache:get_limit(), + ok = file_handle_cache:set_limit(5), + %% Reserves are always accepted, even if above the limit + %% These are for special processes such as quorum queues + ok = file_handle_cache:set_reservation(7), + + Self = self(), + spawn(fun () -> ok = file_handle_cache:obtain(), + Self ! obtained + end), + + Props = file_handle_cache:info([files_reserved, sockets_used]), + ?assertEqual(7, proplists:get_value(files_reserved, Props)), + ?assertEqual(0, proplists:get_value(sockets_used, Props)), + + %% The obtain should still be blocked, as there are no file handles + %% available + receive + obtained -> + throw(error_file_obtained) + after 1000 -> + %% Let's release 5 file handles, that should leave + %% enough free for the `obtain` to go through + file_handle_cache:set_reservation(2), + Props0 = file_handle_cache:info([files_reserved, sockets_used]), + ?assertEqual(2, proplists:get_value(files_reserved, Props0)), + ?assertEqual(1, proplists:get_value(sockets_used, Props0)), + receive + obtained -> + ok = file_handle_cache:set_limit(Limit), + passed + after 5000 -> + throw(error_file_not_released) + end + end. + +file_handle_cache_reserve_release(Config) -> + passed = rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, file_handle_cache_reserve_release1, [Config]). + +file_handle_cache_reserve_release1(_Config) -> + ok = file_handle_cache:set_reservation(7), + ?assertEqual([{files_reserved, 7}], file_handle_cache:info([files_reserved])), + ok = file_handle_cache:set_reservation(3), + ?assertEqual([{files_reserved, 3}], file_handle_cache:info([files_reserved])), + ok = file_handle_cache:release_reservation(), + ?assertEqual([{files_reserved, 0}], file_handle_cache:info([files_reserved])), + passed. + +file_handle_cache_reserve_above_limit(Config) -> + passed = rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, file_handle_cache_reserve_above_limit1, [Config]). + +file_handle_cache_reserve_above_limit1(_Config) -> + Limit = file_handle_cache:get_limit(), + ok = file_handle_cache:set_limit(5), + %% Reserves are always accepted, even if above the limit + %% These are for special processes such as quorum queues + ok = file_handle_cache:obtain(5), + ?assertEqual([{file_descriptor_limit, []}], rabbit_alarm:get_alarms()), + + ok = file_handle_cache:set_reservation(7), + + Props = file_handle_cache:info([files_reserved, sockets_used]), + ?assertEqual(7, proplists:get_value(files_reserved, Props)), + ?assertEqual(5, proplists:get_value(sockets_used, Props)), + + ok = file_handle_cache:set_limit(Limit), + passed. + +file_handle_cache_reserve_open_file_above_limit(Config) -> + passed = rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, file_handle_cache_reserve_open_file_above_limit1, [Config]). + +file_handle_cache_reserve_open_file_above_limit1(_Config) -> + Limit = file_handle_cache:get_limit(), + ok = file_handle_cache:set_limit(5), + %% Reserves are always accepted, even if above the limit + %% These are for special processes such as quorum queues + ok = file_handle_cache:set_reservation(7), + + Self = self(), + TmpDir = filename:join(rabbit_mnesia:dir(), "tmp"), + spawn(fun () -> {ok, _} = file_handle_cache:open( + filename:join(TmpDir, "file_above_limit"), + [write], []), + Self ! opened + end), + + Props = file_handle_cache:info([files_reserved]), + ?assertEqual(7, proplists:get_value(files_reserved, Props)), + + %% The open should still be blocked, as there are no file handles + %% available + receive + opened -> + throw(error_file_opened) + after 1000 -> + %% Let's release 5 file handles, that should leave + %% enough free for the `open` to go through + file_handle_cache:set_reservation(2), + Props0 = file_handle_cache:info([files_reserved, total_used]), + ?assertEqual(2, proplists:get_value(files_reserved, Props0)), + receive + opened -> + ok = file_handle_cache:set_limit(Limit), + passed + after 5000 -> + throw(error_file_not_released) + end + end. + +file_handle_cache_reserve_monitor(Config) -> + passed = rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, file_handle_cache_reserve_monitor1, [Config]). + +file_handle_cache_reserve_monitor1(_Config) -> + %% Check that if the process that does the reserve dies, the file handlers are + %% released by the cache + Self = self(), + Pid = spawn(fun () -> + ok = file_handle_cache:set_reservation(2), + Self ! done, + receive + stop -> ok + end + end), + receive + done -> ok + end, + ?assertEqual([{files_reserved, 2}], file_handle_cache:info([files_reserved])), + Pid ! stop, + timer:sleep(500), + ?assertEqual([{files_reserved, 0}], file_handle_cache:info([files_reserved])), + passed. diff --git a/deps/rabbit/test/unit_gen_server2_SUITE.erl b/deps/rabbit/test/unit_gen_server2_SUITE.erl new file mode 100644 index 0000000000..babd340f19 --- /dev/null +++ b/deps/rabbit/test/unit_gen_server2_SUITE.erl @@ -0,0 +1,152 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2011-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(unit_gen_server2_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-compile(export_all). + +all() -> + [ + {group, sequential_tests} + ]. + +groups() -> + [ + {sequential_tests, [], [ + gen_server2_with_state, + mcall + ]} + ]. + +%% ------------------------------------------------------------------- +%% Testsuite setup/teardown +%% ------------------------------------------------------------------- + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(Group, Config) -> + Config1 = rabbit_ct_helpers:set_config(Config, [ + {rmq_nodename_suffix, Group}, + {rmq_nodes_count, 1} + ]), + rabbit_ct_helpers:run_steps(Config1, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()). + +end_per_group(_Group, Config) -> + rabbit_ct_helpers:run_steps(Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()). + +init_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_started(Config, Testcase). + +end_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_finished(Config, Testcase). + + +%% ------------------------------------------------------------------- +%% Test cases +%% ------------------------------------------------------------------- + +gen_server2_with_state(Config) -> + passed = rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, gen_server2_with_state1, [Config]). + +gen_server2_with_state1(_Config) -> + fhc_state = gen_server2:with_state(file_handle_cache, + fun (S) -> element(1, S) end), + passed. + + +mcall(Config) -> + passed = rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, mcall1, [Config]). + +mcall1(_Config) -> + P1 = spawn(fun gs2_test_listener/0), + register(foo, P1), + global:register_name(gfoo, P1), + + P2 = spawn(fun() -> exit(bang) end), + %% ensure P2 is dead (ignore the race setting up the monitor) + await_exit(P2), + + P3 = spawn(fun gs2_test_crasher/0), + + %% since P2 crashes almost immediately and P3 after receiving its first + %% message, we have to spawn a few more processes to handle the additional + %% cases we're interested in here + register(baz, spawn(fun gs2_test_crasher/0)), + register(bog, spawn(fun gs2_test_crasher/0)), + global:register_name(gbaz, spawn(fun gs2_test_crasher/0)), + + NoNode = rabbit_nodes:make("nonode"), + + Targets = + %% pids + [P1, P2, P3] + ++ + %% registered names + [foo, bar, baz] + ++ + %% {Name, Node} pairs + [{foo, node()}, {bar, node()}, {bog, node()}, {foo, NoNode}] + ++ + %% {global, Name} + [{global, gfoo}, {global, gbar}, {global, gbaz}], + + GoodResults = [{D, goodbye} || D <- [P1, foo, + {foo, node()}, + {global, gfoo}]], + + BadResults = [{P2, noproc}, % died before use + {P3, boom}, % died on first use + {bar, noproc}, % never registered + {baz, boom}, % died on first use + {{bar, node()}, noproc}, % never registered + {{bog, node()}, boom}, % died on first use + {{foo, NoNode}, nodedown}, % invalid node + {{global, gbar}, noproc}, % never registered globally + {{global, gbaz}, boom}], % died on first use + + {Replies, Errors} = gen_server2:mcall([{T, hello} || T <- Targets]), + true = lists:sort(Replies) == lists:sort(GoodResults), + true = lists:sort(Errors) == lists:sort(BadResults), + + %% cleanup (ignore the race setting up the monitor) + P1 ! stop, + await_exit(P1), + passed. + +await_exit(Pid) -> + MRef = erlang:monitor(process, Pid), + receive + {'DOWN', MRef, _, _, _} -> ok + end. + +gs2_test_crasher() -> + receive + {'$gen_call', _From, hello} -> exit(boom) + end. + +gs2_test_listener() -> + receive + {'$gen_call', From, hello} -> + gen_server2:reply(From, goodbye), + gs2_test_listener(); + stop -> + ok + end. diff --git a/deps/rabbit/test/unit_gm_SUITE.erl b/deps/rabbit/test/unit_gm_SUITE.erl new file mode 100644 index 0000000000..74400ddaa5 --- /dev/null +++ b/deps/rabbit/test/unit_gm_SUITE.erl @@ -0,0 +1,242 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(unit_gm_SUITE). + +-behaviour(gm). + +-include_lib("common_test/include/ct.hrl"). + +-include("gm_specs.hrl"). + +-compile(export_all). + +-define(RECEIVE_OR_THROW(Body, Bool, Error), + receive Body -> + true = Bool, + passed + after 5000 -> + throw(Error) + end). + +all() -> + [ + join_leave, + broadcast, + confirmed_broadcast, + member_death, + receive_in_order, + unexpected_msg, + down_in_members_change + ]. + +init_per_suite(Config) -> + ok = application:set_env(mnesia, dir, ?config(priv_dir, Config)), + ok = application:start(mnesia), + {ok, FHC} = file_handle_cache:start_link(), + unlink(FHC), + {ok, WPS} = worker_pool_sup:start_link(), + unlink(WPS), + rabbit_ct_helpers:set_config(Config, [ + {file_handle_cache_pid, FHC}, + {worker_pool_sup_pid, WPS} + ]). + +end_per_suite(Config) -> + exit(?config(worker_pool_sup_pid, Config), shutdown), + exit(?config(file_handle_cache_pid, Config), shutdown), + ok = application:stop(mnesia), + Config. + +%% --------------------------------------------------------------------------- +%% Functional tests +%% --------------------------------------------------------------------------- + +join_leave(_Config) -> + passed = with_two_members(fun (_Pid, _Pid2) -> passed end). + +broadcast(_Config) -> + passed = do_broadcast(fun gm:broadcast/2). + +confirmed_broadcast(_Config) -> + passed = do_broadcast(fun gm:confirmed_broadcast/2). + +member_death(_Config) -> + passed = with_two_members( + fun (Pid, Pid2) -> + {ok, Pid3} = gm:start_link( + ?MODULE, ?MODULE, self(), + fun rabbit_misc:execute_mnesia_transaction/1), + passed = receive_joined(Pid3, [Pid, Pid2, Pid3], + timeout_joining_gm_group_3), + passed = receive_birth(Pid, Pid3, timeout_waiting_for_birth_3_1), + passed = receive_birth(Pid2, Pid3, timeout_waiting_for_birth_3_2), + + unlink(Pid3), + exit(Pid3, kill), + + %% Have to do some broadcasts to ensure that all members + %% find out about the death. + BFun = broadcast_fun(fun gm:confirmed_broadcast/2), + passed = BFun(Pid, Pid2), + passed = BFun(Pid, Pid2), + + passed = receive_death(Pid, Pid3, timeout_waiting_for_death_3_1), + passed = receive_death(Pid2, Pid3, timeout_waiting_for_death_3_2), + + passed + end). + +receive_in_order(_Config) -> + passed = with_two_members( + fun (Pid, Pid2) -> + Numbers = lists:seq(1,1000), + [begin ok = gm:broadcast(Pid, N), ok = gm:broadcast(Pid2, N) end + || N <- Numbers], + passed = receive_numbers( + Pid, Pid, {timeout_for_msgs, Pid, Pid}, Numbers), + passed = receive_numbers( + Pid, Pid2, {timeout_for_msgs, Pid, Pid2}, Numbers), + passed = receive_numbers( + Pid2, Pid, {timeout_for_msgs, Pid2, Pid}, Numbers), + passed = receive_numbers( + Pid2, Pid2, {timeout_for_msgs, Pid2, Pid2}, Numbers), + passed + end). + +unexpected_msg(_Config) -> + passed = with_two_members( + fun(Pid, _) -> + Pid ! {make_ref(), old_gen_server_answer}, + true = erlang:is_process_alive(Pid), + passed + end). + +down_in_members_change(_Config) -> + %% Setup + ok = gm:create_tables(), + {ok, Pid} = gm:start_link(?MODULE, ?MODULE, self(), + fun rabbit_misc:execute_mnesia_transaction/1), + passed = receive_joined(Pid, [Pid], timeout_joining_gm_group_1), + {ok, Pid2} = gm:start_link(?MODULE, ?MODULE, self(), + fun rabbit_misc:execute_mnesia_transaction/1), + passed = receive_joined(Pid2, [Pid, Pid2], timeout_joining_gm_group_2), + passed = receive_birth(Pid, Pid2, timeout_waiting_for_birth_2), + + %% Test. Simulate that the gm group is deleted (forget_group) while + %% processing the 'DOWN' message from the neighbour + process_flag(trap_exit, true), + ok = meck:new(mnesia, [passthrough]), + ok = meck:expect(mnesia, read, fun({gm_group, ?MODULE}) -> + []; + (Key) -> + meck:passthrough([Key]) + end), + gm:leave(Pid2), + Passed = receive + {'EXIT', Pid, shutdown} -> + passed; + {'EXIT', Pid, _} -> + crashed + after 15000 -> + timeout + end, + %% Cleanup + meck:unload(mnesia), + process_flag(trap_exit, false), + passed = Passed. + + +do_broadcast(Fun) -> + with_two_members(broadcast_fun(Fun)). + +broadcast_fun(Fun) -> + fun (Pid, Pid2) -> + ok = Fun(Pid, magic_message), + passed = receive_or_throw({msg, Pid, Pid, magic_message}, + timeout_waiting_for_msg), + passed = receive_or_throw({msg, Pid2, Pid, magic_message}, + timeout_waiting_for_msg) + end. + +with_two_members(Fun) -> + ok = gm:create_tables(), + + {ok, Pid} = gm:start_link(?MODULE, ?MODULE, self(), + fun rabbit_misc:execute_mnesia_transaction/1), + passed = receive_joined(Pid, [Pid], timeout_joining_gm_group_1), + + {ok, Pid2} = gm:start_link(?MODULE, ?MODULE, self(), + fun rabbit_misc:execute_mnesia_transaction/1), + passed = receive_joined(Pid2, [Pid, Pid2], timeout_joining_gm_group_2), + passed = receive_birth(Pid, Pid2, timeout_waiting_for_birth_2), + + passed = Fun(Pid, Pid2), + + ok = gm:leave(Pid), + passed = receive_death(Pid2, Pid, timeout_waiting_for_death_1), + passed = + receive_termination(Pid, normal, timeout_waiting_for_termination_1), + + ok = gm:leave(Pid2), + passed = + receive_termination(Pid2, normal, timeout_waiting_for_termination_2), + + receive X -> throw({unexpected_message, X}) + after 0 -> passed + end. + +receive_or_throw(Pattern, Error) -> + ?RECEIVE_OR_THROW(Pattern, true, Error). + +receive_birth(From, Born, Error) -> + ?RECEIVE_OR_THROW({members_changed, From, Birth, Death}, + ([Born] == Birth) andalso ([] == Death), + Error). + +receive_death(From, Died, Error) -> + ?RECEIVE_OR_THROW({members_changed, From, Birth, Death}, + ([] == Birth) andalso ([Died] == Death), + Error). + +receive_joined(From, Members, Error) -> + ?RECEIVE_OR_THROW({joined, From, Members1}, + lists:usort(Members) == lists:usort(Members1), + Error). + +receive_termination(From, Reason, Error) -> + ?RECEIVE_OR_THROW({termination, From, Reason1}, + Reason == Reason1, + Error). + +receive_numbers(_Pid, _Sender, _Error, []) -> + passed; +receive_numbers(Pid, Sender, Error, [N | Numbers]) -> + ?RECEIVE_OR_THROW({msg, Pid, Sender, M}, + M == N, + Error), + receive_numbers(Pid, Sender, Error, Numbers). + +%% ------------------------------------------------------------------- +%% gm behavior callbacks. +%% ------------------------------------------------------------------- + +joined(Pid, Members) -> + Pid ! {joined, self(), Members}, + ok. + +members_changed(Pid, Births, Deaths) -> + Pid ! {members_changed, self(), Births, Deaths}, + ok. + +handle_msg(Pid, From, Msg) -> + Pid ! {msg, self(), From, Msg}, + ok. + +handle_terminate(Pid, Reason) -> + Pid ! {termination, self(), Reason}, + ok. diff --git a/deps/rabbit/test/unit_log_config_SUITE.erl b/deps/rabbit/test/unit_log_config_SUITE.erl new file mode 100644 index 0000000000..6be403fd3e --- /dev/null +++ b/deps/rabbit/test/unit_log_config_SUITE.erl @@ -0,0 +1,837 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2016-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(unit_log_config_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-compile(export_all). + +all() -> + [ + default, + env_var_tty, + config_file_handler, + config_file_handler_level, + config_file_handler_rotation, + config_console_handler, + config_exchange_handler, + config_syslog_handler, + config_syslog_handler_options, + config_multiple_handlers, + + env_var_overrides_config, + env_var_disable_log, + + config_sinks_level, + config_sink_file, + config_sink_file_override_config_handler_file, + + config_handlers_merged_with_lager_handlers, + sink_handlers_merged_with_lager_extra_sinks_handlers, + sink_file_rewrites_file_backends + ]. + +init_per_testcase(_, Config) -> + application:load(rabbit), + application:load(lager), + application:unset_env(rabbit, log), + application:unset_env(rabbit, lager_log_root), + application:unset_env(rabbit, lager_default_file), + application:unset_env(rabbit, lager_upgrade_file), + application:unset_env(lager, handlers), + application:unset_env(lager, rabbit_handlers), + application:unset_env(lager, extra_sinks), + unset_logs_var_origin(), + Config. + +end_per_testcase(_, Config) -> + application:unset_env(rabbit, log), + application:unset_env(rabbit, lager_log_root), + application:unset_env(rabbit, lager_default_file), + application:unset_env(rabbit, lager_upgrade_file), + application:unset_env(lager, handlers), + application:unset_env(lager, rabbit_handlers), + application:unset_env(lager, extra_sinks), + unset_logs_var_origin(), + application:unload(rabbit), + application:unload(lager), + Config. + +sink_file_rewrites_file_backends(_) -> + application:set_env(rabbit, log, [ + %% Disable rabbit file handler + {file, [{file, false}]}, + {categories, [{federation, [{file, "federation.log"}, {level, warning}]}]} + ]), + + LagerHandlers = [ + {lager_file_backend, [{file, "lager_file.log"}, {level, error}]}, + {lager_file_backend, [{file, "lager_file_1.log"}, {level, error}]}, + {lager_console_backend, [{level, info}]}, + {lager_exchange_backend, [{level, info}]} + ], + application:set_env(lager, handlers, LagerHandlers), + rabbit_lager:configure_lager(), + + ExpectedSinks = sort_sinks(sink_rewrite_sinks()), + ?assertEqual(ExpectedSinks, sort_sinks(application:get_env(lager, extra_sinks, undefined))). + +sink_rewrite_sinks() -> + [{error_logger_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,info]}]}, + {rabbit_handlers,[{lager_forwarder_backend,[lager_event,info]}]}]}, + {rabbit_log_channel_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,info]}]}, + {rabbit_handlers,[{lager_forwarder_backend,[lager_event,info]}]}]}, + {rabbit_log_connection_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,info]}]}, + {rabbit_handlers,[{lager_forwarder_backend,[lager_event,info]}]}]}, + {rabbit_log_feature_flags_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,info]}]}, + {rabbit_handlers,[{lager_forwarder_backend,[lager_event,info]}]}]}, + {rabbit_log_federation_lager_event, + [{handlers,[ + {lager_file_backend, + [{date, ""}, + {file, "federation.log"}, + {formatter_config, formatter_config(file)}, + {level, warning}, + {size, 0}]}, + {lager_console_backend, [{level, warning}]}, + {lager_exchange_backend, [{level, warning}]} + ]}, + {rabbit_handlers,[ + {lager_file_backend, + [{date, ""}, + {file, "federation.log"}, + {formatter_config, formatter_config(file)}, + {level, warning}, + {size, 0}]}, + {lager_console_backend, [{level, warning}]}, + {lager_exchange_backend, [{level, warning}]} + ]}]}, + {rabbit_log_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,info]}]}, + {rabbit_handlers,[{lager_forwarder_backend,[lager_event,info]}]}]}, + {rabbit_log_ldap_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,info]}]}, + {rabbit_handlers, + [{lager_forwarder_backend,[lager_event,info]}]}]}, + {rabbit_log_mirroring_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,info]}]}, + {rabbit_handlers,[{lager_forwarder_backend,[lager_event,info]}]}]}, + {rabbit_log_osiris_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,info]}]}, + {rabbit_handlers, + [{lager_forwarder_backend,[lager_event,info]}]}]}, + {rabbit_log_prelaunch_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,info]}]}, + {rabbit_handlers,[{lager_forwarder_backend,[lager_event,info]}]}]}, + {rabbit_log_queue_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,info]}]}, + {rabbit_handlers,[{lager_forwarder_backend,[lager_event,info]}]}]}, + {rabbit_log_ra_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,info]}]}, + {rabbit_handlers, + [{lager_forwarder_backend,[lager_event,info]}]}]}, + {rabbit_log_shovel_lager_event, + [{handlers, [{lager_forwarder_backend,[lager_event,info]}]}, + {rabbit_handlers, + [{lager_forwarder_backend,[lager_event,info]}]}]}, + {rabbit_log_upgrade_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,info]}]}, + {rabbit_handlers,[{lager_forwarder_backend,[lager_event,info]}]}]} + ]. + +sink_handlers_merged_with_lager_extra_sinks_handlers(_) -> + DefaultLevel = debug, + application:set_env(rabbit, log, [ + {file, [{file, "rabbit_file.log"}, {level, DefaultLevel}]}, + {console, [{enabled, true}, {level, error}]}, + {exchange, [{enabled, true}, {level, error}]}, + {categories, [ + {connection, [{level, debug}]}, + {channel, [{level, warning}, {file, "channel_log.log"}]} + ]} + ]), + + LagerSinks = [ + {rabbit_log_connection_lager_event, + [{handlers, + [{lager_file_backend, + [{file, "connection_lager.log"}, + {level, info}]}]}]}, + {rabbit_log_channel_lager_event, + [{handlers, + [{lager_console_backend, [{level, debug}]}, + {lager_exchange_backend, [{level, debug}]}, + {lager_file_backend, [{level, error}, + {file, "channel_lager.log"}]}]}]}], + + application:set_env(lager, extra_sinks, LagerSinks), + rabbit_lager:configure_lager(), + + ExpectedSinks = sort_sinks([ + {error_logger_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,DefaultLevel]}]}, + {rabbit_handlers,[{lager_forwarder_backend,[lager_event,DefaultLevel]}]}]}, + {rabbit_log_channel_lager_event, + [{handlers,[ + {lager_console_backend, [{level, error}, + {formatter_config, formatter_config(console)}]}, + {lager_exchange_backend, [{level, error}, + {formatter_config, formatter_config(exchange)}]}, + {lager_file_backend, + [{date, ""}, + {file, "channel_log.log"}, + {formatter_config, formatter_config(file)}, + {level, warning}, + {size, 0}]}, + {lager_console_backend, [{level, debug}]}, + {lager_exchange_backend, [{level, debug}]}, + {lager_file_backend, [{level, error}, + {file, "channel_lager.log"}]} + ]}, + {rabbit_handlers,[ + {lager_console_backend, [{level, error}, + {formatter_config, formatter_config(console)}]}, + {lager_exchange_backend, [{level, error}, + {formatter_config, formatter_config(exchange)}]}, + {lager_file_backend, + [{date, ""}, + {file, "channel_log.log"}, + {formatter_config, formatter_config(file)}, + {level, warning}, + {size, 0}]}]} + ]}, + {rabbit_log_connection_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,debug]}, + {lager_file_backend, [{file, "connection_lager.log"}, {level, info}]}]}, + {rabbit_handlers,[{lager_forwarder_backend,[lager_event,debug]}]}]}, + {rabbit_log_feature_flags_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,DefaultLevel]}]}, + {rabbit_handlers,[{lager_forwarder_backend,[lager_event,DefaultLevel]}]}]}, + {rabbit_log_federation_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,DefaultLevel]}]}, + {rabbit_handlers,[{lager_forwarder_backend,[lager_event,DefaultLevel]}]}]}, + {rabbit_log_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,DefaultLevel]}]}, + {rabbit_handlers,[{lager_forwarder_backend,[lager_event,DefaultLevel]}]}]}, + {rabbit_log_ldap_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,DefaultLevel]}]}, + {rabbit_handlers, + [{lager_forwarder_backend,[lager_event,DefaultLevel]}]}]}, + {rabbit_log_mirroring_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,DefaultLevel]}]}, + {rabbit_handlers,[{lager_forwarder_backend,[lager_event,DefaultLevel]}]}]}, + {rabbit_log_osiris_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,DefaultLevel]}]}, + {rabbit_handlers, + [{lager_forwarder_backend,[lager_event,DefaultLevel]}]}]}, + {rabbit_log_prelaunch_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,DefaultLevel]}]}, + {rabbit_handlers,[{lager_forwarder_backend,[lager_event,DefaultLevel]}]}]}, + {rabbit_log_queue_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,DefaultLevel]}]}, + {rabbit_handlers,[{lager_forwarder_backend,[lager_event,DefaultLevel]}]}]}, + {rabbit_log_ra_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,DefaultLevel]}]}, + {rabbit_handlers, + [{lager_forwarder_backend,[lager_event,DefaultLevel]}]}]}, + {rabbit_log_shovel_lager_event, + [{handlers, [{lager_forwarder_backend,[lager_event,DefaultLevel]}]}, + {rabbit_handlers, + [{lager_forwarder_backend,[lager_event,DefaultLevel]}]}]}, + {rabbit_log_upgrade_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,DefaultLevel]}]}, + {rabbit_handlers,[{lager_forwarder_backend,[lager_event,DefaultLevel]}]}]}]), + + ?assertEqual(ExpectedSinks, sort_sinks(application:get_env(lager, extra_sinks, undefined))). + +config_handlers_merged_with_lager_handlers(_) -> + application:set_env(rabbit, log, [ + {file, [{file, "rabbit_file.log"}, {level, debug}]}, + {console, [{enabled, true}, {level, error}]}, + {exchange, [{enabled, true}, {level, error}]}, + {syslog, [{enabled, true}]} + ]), + + LagerHandlers = [ + {lager_file_backend, [{file, "lager_file.log"}, {level, info}]}, + {lager_console_backend, [{level, info}]}, + {lager_exchange_backend, [{level, info}]}, + {lager_exchange_backend, [{level, info}]} + ], + application:set_env(lager, handlers, LagerHandlers), + rabbit_lager:configure_lager(), + + FileHandlers = default_expected_handlers("rabbit_file.log", debug), + ConsoleHandlers = expected_console_handler(error), + RabbitHandlers = expected_rabbit_handler(error), + SyslogHandlers = expected_syslog_handler(), + + ExpectedRabbitHandlers = sort_handlers(FileHandlers ++ ConsoleHandlers ++ RabbitHandlers ++ SyslogHandlers), + ExpectedHandlers = sort_handlers(ExpectedRabbitHandlers ++ LagerHandlers), + + ?assertEqual(ExpectedRabbitHandlers, sort_handlers(application:get_env(lager, rabbit_handlers, undefined))), + ?assertEqual(ExpectedHandlers, sort_handlers(application:get_env(lager, handlers, undefined))). + +config_sinks_level(_) -> + DefaultLogFile = "rabbit_default.log", + application:set_env(rabbit, lager_default_file, DefaultLogFile), + + application:set_env(rabbit, log, [ + {categories, [ + {connection, [{level, warning}]}, + {channel, [{level, debug}]}, + {mirroring, [{level, error}]} + ]} + ]), + + rabbit_lager:configure_lager(), + + ExpectedSinks = sort_sinks(level_sinks()), + ?assertEqual(ExpectedSinks, sort_sinks(application:get_env(lager, extra_sinks, undefined))). + +level_sinks() -> + [{error_logger_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,info]}]}, + {rabbit_handlers,[{lager_forwarder_backend,[lager_event,info]}]}]}, + {rabbit_log_channel_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,debug]}]}, + {rabbit_handlers,[{lager_forwarder_backend,[lager_event,debug]}]}]}, + {rabbit_log_connection_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,warning]}]}, + {rabbit_handlers,[{lager_forwarder_backend,[lager_event,warning]}]}]}, + {rabbit_log_feature_flags_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,info]}]}, + {rabbit_handlers,[{lager_forwarder_backend,[lager_event,info]}]}]}, + {rabbit_log_federation_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,info]}]}, + {rabbit_handlers,[{lager_forwarder_backend,[lager_event,info]}]}]}, + {rabbit_log_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,info]}]}, + {rabbit_handlers,[{lager_forwarder_backend,[lager_event,info]}]}]}, + {rabbit_log_ldap_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,info]}]}, + {rabbit_handlers, + [{lager_forwarder_backend,[lager_event,info]}]}]}, + {rabbit_log_mirroring_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,error]}]}, + {rabbit_handlers,[{lager_forwarder_backend,[lager_event,error]}]}]}, + {rabbit_log_osiris_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,info]}]}, + {rabbit_handlers, + [{lager_forwarder_backend,[lager_event,info]}]}]}, + {rabbit_log_prelaunch_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,info]}]}, + {rabbit_handlers,[{lager_forwarder_backend,[lager_event,info]}]}]}, + {rabbit_log_queue_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,info]}]}, + {rabbit_handlers,[{lager_forwarder_backend,[lager_event,info]}]}]}, + {rabbit_log_ra_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,info]}]}, + {rabbit_handlers, + [{lager_forwarder_backend,[lager_event,info]}]}]}, + {rabbit_log_shovel_lager_event, + [{handlers, [{lager_forwarder_backend,[lager_event,info]}]}, + {rabbit_handlers, + [{lager_forwarder_backend, + [lager_event,info]}]}]}, + {rabbit_log_upgrade_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,info]}]}, + {rabbit_handlers,[{lager_forwarder_backend,[lager_event,info]}]}]} + ]. + +config_sink_file(_) -> + DefaultLogFile = "rabbit_default.log", + application:set_env(rabbit, lager_default_file, DefaultLogFile), + + DefaultLevel = error, + application:set_env(rabbit, log, [ + {console, [{enabled, true}]}, + {exchange, [{enabled, true}]}, + {file, [{level, DefaultLevel}]}, + {categories, [ + {connection, [{file, "connection.log"}, {level, warning}]} + ]} + ]), + + rabbit_lager:configure_lager(), + + ExpectedSinks = sort_sinks(file_sinks(DefaultLevel)), + ?assertEqual(ExpectedSinks, sort_sinks(application:get_env(lager, extra_sinks, undefined))). + +config_sink_file_override_config_handler_file(_) -> + DefaultLogFile = "rabbit_default.log", + application:set_env(rabbit, lager_default_file, DefaultLogFile), + + NonDefaultLogFile = "rabbit_not_default.log", + + DefaultLevel = error, + application:set_env(rabbit, log, [ + {file, [{file, NonDefaultLogFile}, {level, DefaultLevel}]}, + {console, [{enabled, true}]}, + {exchange, [{enabled, true}]}, + {categories, [ + {connection, [{file, "connection.log"}, {level, warning}]} + ]} + ]), + + rabbit_lager:configure_lager(), + + ExpectedSinks = sort_sinks(file_sinks(DefaultLevel)), + ?assertEqual(ExpectedSinks, sort_sinks(application:get_env(lager, extra_sinks, undefined))). + +file_sinks() -> + file_sinks(info). + +file_sinks(DefaultLevel) -> + [{error_logger_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,DefaultLevel]}]}, + {rabbit_handlers,[{lager_forwarder_backend,[lager_event,DefaultLevel]}]}]}, + {rabbit_log_channel_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,DefaultLevel]}]}, + {rabbit_handlers,[{lager_forwarder_backend,[lager_event,DefaultLevel]}]}]}, + {rabbit_log_connection_lager_event, + [{handlers,[ + {lager_console_backend, [{level, warning}, + {formatter_config, formatter_config(console)}]}, + {lager_exchange_backend, [{level, warning}, + {formatter_config, formatter_config(exchange)}]}, + {lager_file_backend, + [{date, ""}, + {file, "connection.log"}, + {formatter_config, formatter_config(file)}, + {level, error}, + {size, 0}]}]}, + {rabbit_handlers,[ + {lager_console_backend, [{level, warning}, + {formatter_config, formatter_config(console)}]}, + {lager_exchange_backend, [{level, warning}, + {formatter_config, formatter_config(exchange)}]}, + {lager_file_backend, + [{date, ""}, + {file, "connection.log"}, + {formatter_config, formatter_config(backend)}, + {level, error}, + {size, 0}]}]} + ]}, + {rabbit_log_feature_flags_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,DefaultLevel]}]}, + {rabbit_handlers,[{lager_forwarder_backend,[lager_event,DefaultLevel]}]}]}, + {rabbit_log_federation_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,DefaultLevel]}]}, + {rabbit_handlers,[{lager_forwarder_backend,[lager_event,DefaultLevel]}]}]}, + {rabbit_log_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,DefaultLevel]}]}, + {rabbit_handlers,[{lager_forwarder_backend,[lager_event,DefaultLevel]}]}]}, + {rabbit_log_ldap_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,DefaultLevel]}]}, + {rabbit_handlers, + [{lager_forwarder_backend,[lager_event,DefaultLevel]}]}]}, + {rabbit_log_mirroring_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,DefaultLevel]}]}, + {rabbit_handlers,[{lager_forwarder_backend,[lager_event,DefaultLevel]}]}]}, + {rabbit_log_osiris_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,DefaultLevel]}]}, + {rabbit_handlers, + [{lager_forwarder_backend,[lager_event,DefaultLevel]}]}]}, + {rabbit_log_prelaunch_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,DefaultLevel]}]}, + {rabbit_handlers,[{lager_forwarder_backend,[lager_event,DefaultLevel]}]}]}, + {rabbit_log_queue_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,DefaultLevel]}]}, + {rabbit_handlers,[{lager_forwarder_backend,[lager_event,DefaultLevel]}]}]}, + {rabbit_log_ra_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,DefaultLevel]}]}, + {rabbit_handlers, + [{lager_forwarder_backend,[lager_event,DefaultLevel]}]}]}, + {rabbit_log_shovel_lager_event, + [{handlers, [{lager_forwarder_backend,[lager_event,DefaultLevel]}]}, + {rabbit_handlers, + [{lager_forwarder_backend,[lager_event,DefaultLevel]}]}]}, + {rabbit_log_upgrade_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,DefaultLevel]}]}, + {rabbit_handlers,[{lager_forwarder_backend,[lager_event,DefaultLevel]}]}]} + ]. + +config_multiple_handlers(_) -> + DefaultLogFile = "rabbit_default.log", + application:set_env(rabbit, lager_default_file, DefaultLogFile), + + application:set_env(rabbit, log, [ + %% Disable file output + {file, [{file, false}]}, + %% Enable console output + {console, [{enabled, true}]}, + %% Enable exchange output + {exchange, [{enabled, true}]}, + %% Enable a syslog output + {syslog, [{enabled, true}, {level, error}]}]), + + rabbit_lager:configure_lager(), + + ConsoleHandlers = expected_console_handler(), + RabbitHandlers = expected_rabbit_handler(), + SyslogHandlers = expected_syslog_handler(error), + + ExpectedHandlers = sort_handlers(SyslogHandlers ++ ConsoleHandlers ++ RabbitHandlers), + + ?assertEqual(ExpectedHandlers, sort_handlers(application:get_env(lager, handlers, undefined))), + ?assertEqual(ExpectedHandlers, sort_handlers(application:get_env(lager, rabbit_handlers, undefined))). + +config_console_handler(_) -> + DefaultLogFile = "rabbit_default.log", + application:set_env(rabbit, lager_default_file, DefaultLogFile), + application:set_env(rabbit, log, [{console, [{enabled, true}]}]), + + rabbit_lager:configure_lager(), + + FileHandlers = default_expected_handlers(DefaultLogFile), + ConsoleHandlers = expected_console_handler(), + + ExpectedHandlers = sort_handlers(FileHandlers ++ ConsoleHandlers), + + ?assertEqual(ExpectedHandlers, sort_handlers(application:get_env(lager, handlers, undefined))), + ?assertEqual(ExpectedHandlers, sort_handlers(application:get_env(lager, rabbit_handlers, undefined))). + +config_exchange_handler(_) -> + DefaultLogFile = "rabbit_default.log", + application:set_env(rabbit, lager_default_file, DefaultLogFile), + application:set_env(rabbit, log, [{exchange, [{enabled, true}]}]), + + rabbit_lager:configure_lager(), + + FileHandlers = default_expected_handlers(DefaultLogFile), + ExchangeHandlers = expected_rabbit_handler(), + + ExpectedHandlers = sort_handlers(FileHandlers ++ ExchangeHandlers), + + ?assertEqual(ExpectedHandlers, sort_handlers(application:get_env(lager, handlers, undefined))), + ?assertEqual(ExpectedHandlers, sort_handlers(application:get_env(lager, rabbit_handlers, undefined))). + +expected_console_handler() -> + expected_console_handler(debug). + +expected_console_handler(Level) -> + [{lager_console_backend, [{level, Level}, + {formatter_config, formatter_config(console)}]}]. + +expected_rabbit_handler() -> + expected_rabbit_handler(debug). + +expected_rabbit_handler(Level) -> + [{lager_exchange_backend, [{level, Level}, + {formatter_config, formatter_config(exchange)}]}]. + +config_syslog_handler(_) -> + DefaultLogFile = "rabbit_default.log", + application:set_env(rabbit, lager_default_file, DefaultLogFile), + application:set_env(rabbit, log, [{syslog, [{enabled, true}]}]), + + rabbit_lager:configure_lager(), + + FileHandlers = default_expected_handlers(DefaultLogFile), + SyslogHandlers = expected_syslog_handler(), + + ExpectedHandlers = sort_handlers(FileHandlers ++ SyslogHandlers), + + ?assertEqual(ExpectedHandlers, sort_handlers(application:get_env(lager, handlers, undefined))), + ?assertEqual(ExpectedHandlers, sort_handlers(application:get_env(lager, rabbit_handlers, undefined))). + +config_syslog_handler_options(_) -> + DefaultLogFile = "rabbit_default.log", + application:set_env(rabbit, lager_default_file, DefaultLogFile), + application:set_env(rabbit, log, [{syslog, [{enabled, true}, + {level, warning}]}]), + + rabbit_lager:configure_lager(), + + FileHandlers = default_expected_handlers(DefaultLogFile), + SyslogHandlers = expected_syslog_handler(warning), + + ExpectedHandlers = sort_handlers(FileHandlers ++ SyslogHandlers), + + ?assertEqual(ExpectedHandlers, sort_handlers(application:get_env(lager, handlers, undefined))), + ?assertEqual(ExpectedHandlers, sort_handlers(application:get_env(lager, rabbit_handlers, undefined))). + +expected_syslog_handler() -> + expected_syslog_handler(debug). + +expected_syslog_handler(Level) -> + [{syslog_lager_backend, [Level, + {}, + {lager_default_formatter, syslog_formatter_config()}]}]. + +env_var_overrides_config(_) -> + EnvLogFile = "rabbit_default.log", + application:set_env(rabbit, lager_default_file, EnvLogFile), + + ConfigLogFile = "rabbit_not_default.log", + application:set_env(rabbit, log, [{file, [{file, ConfigLogFile}]}]), + + set_logs_var_origin(environment), + rabbit_lager:configure_lager(), + + ExpectedHandlers = default_expected_handlers(EnvLogFile), + ?assertEqual(ExpectedHandlers, sort_handlers(application:get_env(lager, handlers, undefined))), + ?assertEqual(ExpectedHandlers, sort_handlers(application:get_env(lager, rabbit_handlers, undefined))). + +env_var_disable_log(_) -> + application:set_env(rabbit, lager_default_file, false), + + ConfigLogFile = "rabbit_not_default.log", + application:set_env(rabbit, log, [{file, [{file, ConfigLogFile}]}]), + + set_logs_var_origin(environment), + rabbit_lager:configure_lager(), + + ExpectedHandlers = [], + ?assertEqual(ExpectedHandlers, sort_handlers(application:get_env(lager, handlers, undefined))), + ?assertEqual(ExpectedHandlers, sort_handlers(application:get_env(lager, rabbit_handlers, undefined))). + +config_file_handler(_) -> + DefaultLogFile = "rabbit_default.log", + application:set_env(rabbit, lager_default_file, DefaultLogFile), + + NonDefaultLogFile = "rabbit_not_default.log", + application:set_env(rabbit, log, [{file, [{file, NonDefaultLogFile}]}]), + + rabbit_lager:configure_lager(), + + ExpectedHandlers = default_expected_handlers(NonDefaultLogFile), + ?assertEqual(ExpectedHandlers, sort_handlers(application:get_env(lager, handlers, undefined))), + ?assertEqual(ExpectedHandlers, sort_handlers(application:get_env(lager, rabbit_handlers, undefined))). + +config_file_handler_level(_) -> + DefaultLogFile = "rabbit_default.log", + application:set_env(rabbit, lager_default_file, DefaultLogFile), + + application:set_env(rabbit, log, [{file, [{level, warning}]}]), + rabbit_lager:configure_lager(), + + ExpectedHandlers = default_expected_handlers(DefaultLogFile, warning), + ?assertEqual(ExpectedHandlers, sort_handlers(application:get_env(lager, handlers, undefined))), + ?assertEqual(ExpectedHandlers, sort_handlers(application:get_env(lager, rabbit_handlers, undefined))). + +config_file_handler_rotation(_) -> + DefaultLogFile = "rabbit_default.log", + application:set_env(rabbit, lager_default_file, DefaultLogFile), + + application:set_env(rabbit, log, [{file, [{date, "$D0"}, {size, 5000}, {count, 10}]}]), + rabbit_lager:configure_lager(), + + ExpectedHandlers = sort_handlers(default_expected_handlers(DefaultLogFile, debug, 5000, "$D0", [{count, 10}])), + ?assertEqual(ExpectedHandlers, sort_handlers(application:get_env(lager, handlers, undefined))), + ?assertEqual(ExpectedHandlers, sort_handlers(application:get_env(lager, rabbit_handlers, undefined))). + +default(_) -> + LogRoot = "/tmp/log_base", + application:set_env(rabbit, lager_log_root, LogRoot), + LogFile = "rabbit_default.log", + application:set_env(rabbit, lager_default_file, LogFile), + LogUpgradeFile = "rabbit_default_upgrade.log", + application:set_env(rabbit, lager_upgrade_file, LogUpgradeFile), + + rabbit_lager:configure_lager(), + + ExpectedHandlers = default_expected_handlers(LogFile), + LogRoot = application:get_env(lager, log_root, undefined), + ?assertEqual(ExpectedHandlers, sort_handlers(application:get_env(lager, handlers, undefined))), + ?assertEqual(ExpectedHandlers, sort_handlers(application:get_env(lager, rabbit_handlers, undefined))), + + ExpectedSinks = default_expected_sinks(LogUpgradeFile), + ?assertEqual(ExpectedSinks, sort_sinks(application:get_env(lager, extra_sinks, undefined))). + +default_expected_handlers(File) -> + default_expected_handlers(File, debug, 0, ""). +default_expected_handlers(File, Level) -> + default_expected_handlers(File, Level, 0, ""). +default_expected_handlers(File, Level, RotSize, RotDate) -> + default_expected_handlers(File, Level, RotSize, RotDate, []). +default_expected_handlers(File, Level, RotSize, RotDate, Extra) -> + [{lager_file_backend, + [{date, RotDate}, + {file, File}, + {formatter_config, formatter_config(file)}, + {level, Level}, + {size, RotSize}] ++ Extra}]. + +default_expected_sinks(UpgradeFile) -> + [{error_logger_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,info]}]}, + {rabbit_handlers,[{lager_forwarder_backend,[lager_event,info]}]}]}, + {rabbit_log_channel_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,info]}]}, + {rabbit_handlers,[{lager_forwarder_backend,[lager_event,info]}]}]}, + {rabbit_log_connection_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,info]}]}, + {rabbit_handlers,[{lager_forwarder_backend,[lager_event,info]}]}]}, + {rabbit_log_feature_flags_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,info]}]}, + {rabbit_handlers,[{lager_forwarder_backend,[lager_event,info]}]}]}, + {rabbit_log_federation_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,info]}]}, + {rabbit_handlers,[{lager_forwarder_backend,[lager_event,info]}]}]}, + {rabbit_log_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,info]}]}, + {rabbit_handlers,[{lager_forwarder_backend,[lager_event,info]}]}]}, + {rabbit_log_ldap_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,info]}]}, + {rabbit_handlers, + [{lager_forwarder_backend,[lager_event,info]}]}]}, + {rabbit_log_mirroring_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,info]}]}, + {rabbit_handlers,[{lager_forwarder_backend,[lager_event,info]}]}]}, + {rabbit_log_osiris_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,info]}]}, + {rabbit_handlers, + [{lager_forwarder_backend,[lager_event,info]}]}]}, + {rabbit_log_prelaunch_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,info]}]}, + {rabbit_handlers,[{lager_forwarder_backend,[lager_event,info]}]}]}, + {rabbit_log_queue_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,info]}]}, + {rabbit_handlers,[{lager_forwarder_backend,[lager_event,info]}]}]}, + {rabbit_log_ra_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,info]}]}, + {rabbit_handlers, + [{lager_forwarder_backend,[lager_event,info]}]}]}, + {rabbit_log_shovel_lager_event, + [{handlers, [{lager_forwarder_backend,[lager_event,info]}]}, + {rabbit_handlers, + [{lager_forwarder_backend,[lager_event,info]}]}]}, + {rabbit_log_upgrade_lager_event, + [{handlers, + [{lager_file_backend, + [{date,[]}, + {file, UpgradeFile}, + {formatter_config, formatter_config(file)}, + {level,info}, + {size,0}]}]}, + {rabbit_handlers, + [{lager_file_backend, + [{date,[]}, + {file, UpgradeFile}, + {formatter_config, formatter_config(file)}, + {level,info}, + {size,0}]}]}]}]. + +env_var_tty(_) -> + application:set_env(rabbit, lager_log_root, "/tmp/log_base"), + application:set_env(rabbit, lager_default_file, tty), + application:set_env(rabbit, lager_upgrade_file, tty), + %% tty can only be set explicitly + set_logs_var_origin(environment), + + rabbit_lager:configure_lager(), + + ExpectedHandlers = tty_expected_handlers(), + ?assertEqual(ExpectedHandlers, sort_handlers(application:get_env(lager, handlers, undefined))), + ?assertEqual(ExpectedHandlers, sort_handlers(application:get_env(lager, rabbit_handlers, undefined))), + + %% Upgrade sink will be different. + ExpectedSinks = tty_expected_sinks(), + ?assertEqual(ExpectedSinks, sort_sinks(application:get_env(lager, extra_sinks, undefined))). + +set_logs_var_origin(Origin) -> + Context = #{var_origins => #{main_log_file => Origin}}, + rabbit_prelaunch:store_context(Context), + ok. + +unset_logs_var_origin() -> + rabbit_prelaunch:clear_context_cache(), + ok. + +tty_expected_handlers() -> + [{lager_console_backend, + [{formatter_config, formatter_config(console)}, + {level, debug}]}]. + +tty_expected_sinks() -> + [{error_logger_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,info]}]}, + {rabbit_handlers,[{lager_forwarder_backend,[lager_event,info]}]}]}, + {rabbit_log_channel_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,info]}]}, + {rabbit_handlers,[{lager_forwarder_backend,[lager_event,info]}]}]}, + {rabbit_log_connection_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,info]}]}, + {rabbit_handlers,[{lager_forwarder_backend,[lager_event,info]}]}]}, + {rabbit_log_feature_flags_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,info]}]}, + {rabbit_handlers,[{lager_forwarder_backend,[lager_event,info]}]}]}, + {rabbit_log_federation_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,info]}]}, + {rabbit_handlers,[{lager_forwarder_backend,[lager_event,info]}]}]}, + {rabbit_log_lager_event, + [{handlers, [{lager_forwarder_backend,[lager_event,info]}]}, + {rabbit_handlers, [{lager_forwarder_backend,[lager_event,info]}]}]}, + {rabbit_log_ldap_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,info]}]}, + {rabbit_handlers, + [{lager_forwarder_backend,[lager_event,info]}]}]}, + {rabbit_log_mirroring_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,info]}]}, + {rabbit_handlers,[{lager_forwarder_backend,[lager_event,info]}]}]}, + {rabbit_log_osiris_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,info]}]}, + {rabbit_handlers, + [{lager_forwarder_backend,[lager_event,info]}]}]}, + {rabbit_log_prelaunch_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,info]}]}, + {rabbit_handlers,[{lager_forwarder_backend,[lager_event,info]}]}]}, + {rabbit_log_queue_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,info]}]}, + {rabbit_handlers,[{lager_forwarder_backend,[lager_event,info]}]}]}, + {rabbit_log_ra_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,info]}]}, + {rabbit_handlers, + [{lager_forwarder_backend,[lager_event,info]}]}]}, + {rabbit_log_shovel_lager_event, + [{handlers, [{lager_forwarder_backend,[lager_event,info]}]}, + {rabbit_handlers, + [{lager_forwarder_backend,[lager_event,info]}]}]}, + {rabbit_log_upgrade_lager_event, + [{handlers,[{lager_forwarder_backend,[lager_event,info]}]}, + {rabbit_handlers,[{lager_forwarder_backend,[lager_event,info]}]}]}]. + +sort_sinks(Sinks) -> + lists:ukeysort(1, + lists:map( + fun({Name, Config}) -> + Handlers = proplists:get_value(handlers, Config), + RabbitHandlers = proplists:get_value(rabbit_handlers, Config), + {Name, lists:ukeymerge(1, + [{handlers, sort_handlers(Handlers)}, + {rabbit_handlers, sort_handlers(RabbitHandlers)}], + lists:ukeysort(1, Config))} + end, + Sinks)). + +sort_handlers(Handlers) -> + lists:keysort(1, + lists:map( + fun + ({Name, [{Atom, _}|_] = Config}) when is_atom(Atom) -> + {Name, lists:ukeysort(1, Config)}; + %% Non-proplist configuration. forwarder backend + (Other) -> + Other + end, + Handlers)). + +formatter_config(console) -> + [date," ",time," ",color,"[",severity, "] ", {pid,[]}, " ",message,"\r\n"]; +formatter_config(_) -> + [date," ",time," ",color,"[",severity, "] ", {pid,[]}, " ",message,"\n"]. + +syslog_formatter_config() -> + [color,"[",severity, "] ", {pid,[]}, " ",message,"\n"]. diff --git a/deps/rabbit/test/unit_log_management_SUITE.erl b/deps/rabbit/test/unit_log_management_SUITE.erl new file mode 100644 index 0000000000..9fc9c7839d --- /dev/null +++ b/deps/rabbit/test/unit_log_management_SUITE.erl @@ -0,0 +1,413 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2011-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(unit_log_management_SUITE). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("kernel/include/file.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). + +-compile(export_all). + +-define(TIMEOUT, 30000). + +all() -> + [ + {group, non_parallel_tests} + ]. + +groups() -> + [ + {non_parallel_tests, [], [ + log_management, + log_file_initialised_during_startup, + log_file_fails_to_initialise_during_startup, + externally_rotated_logs_are_automatically_reopened + ]} + ]. + +%% ------------------------------------------------------------------- +%% Testsuite setup/teardown. +%% ------------------------------------------------------------------- + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(Group, Config) -> + Config1 = rabbit_ct_helpers:set_config(Config, [ + {rmq_nodename_suffix, Group}, + {rmq_nodes_count, 2} + ]), + rabbit_ct_helpers:run_steps(Config1, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()). + +end_per_group(_Group, Config) -> + rabbit_ct_helpers:run_steps(Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()). + +init_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_started(Config, Testcase). + +end_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_finished(Config, Testcase). + +%% ------------------------------------------------------------------- +%% Application management. +%% ------------------------------------------------------------------- + +app_management(Config) -> + passed = rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, app_management1, [Config]). + +app_management1(_Config) -> + wait_for_application(rabbit), + %% Starting, stopping and diagnostics. Note that we don't try + %% 'report' when the rabbit app is stopped and that we enable + %% tracing for the duration of this function. + ok = rabbit_trace:start(<<"/">>), + ok = rabbit:stop(), + ok = rabbit:stop(), + ok = no_exceptions(rabbit, status, []), + ok = no_exceptions(rabbit, environment, []), + ok = rabbit:start(), + ok = rabbit:start(), + ok = no_exceptions(rabbit, status, []), + ok = no_exceptions(rabbit, environment, []), + ok = rabbit_trace:stop(<<"/">>), + passed. + +no_exceptions(Mod, Fun, Args) -> + try erlang:apply(Mod, Fun, Args) of _ -> ok + catch Type:Ex -> {Type, Ex} + end. + +wait_for_application(Application) -> + wait_for_application(Application, 5000). + +wait_for_application(_, Time) when Time =< 0 -> + {error, timeout}; +wait_for_application(Application, Time) -> + Interval = 100, + case lists:keyfind(Application, 1, application:which_applications()) of + false -> + timer:sleep(Interval), + wait_for_application(Application, Time - Interval); + _ -> ok + end. + + + +%% ------------------------------------------------------------------- +%% Log management. +%% ------------------------------------------------------------------- + +log_management(Config) -> + passed = rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, log_management1, [Config]). + +log_management1(_Config) -> + [LogFile|_] = rabbit:log_locations(), + Suffix = ".0", + + ok = test_logs_working([LogFile]), + + %% prepare basic logs + file:delete(LogFile ++ Suffix), + ok = test_logs_working([LogFile]), + + %% simple log rotation + ok = rabbit:rotate_logs(), + %% rabbit:rotate_logs/0 is asynchronous due to a limitation in + %% Lager. Therefore, we have no choice but to wait an arbitrary + %% amount of time. + ok = rabbit_ct_helpers:await_condition( + fun() -> + [true, true] =:= + non_empty_files([LogFile ++ Suffix, LogFile]) + end, 5000), + ok = test_logs_working([LogFile]), + + %% log rotation on empty files + ok = clean_logs([LogFile], Suffix), + ok = rabbit:rotate_logs(), + ok = rabbit_ct_helpers:await_condition( + fun() -> + [true, true] =:= + non_empty_files([LogFile ++ Suffix, LogFile]) + end, 5000), + + %% logs with suffix are not writable + ok = rabbit:rotate_logs(), + ok = rabbit_ct_helpers:await_condition( + fun() -> + ok =:= make_files_non_writable([LogFile ++ Suffix]) + end, 5000), + ok = rabbit:rotate_logs(), + ok = rabbit_ct_helpers:await_condition( + fun() -> + ok =:= test_logs_working([LogFile]) + end, 5000), + + %% rotate when original log files are not writable + ok = make_files_non_writable([LogFile]), + ok = rabbit:rotate_logs(), + timer:sleep(2000), + + %% logging directed to tty (first, remove handlers) + ok = rabbit:stop(), + ok = make_files_writable([LogFile ++ Suffix]), + ok = clean_logs([LogFile], Suffix), + ok = application:set_env(rabbit, lager_default_file, tty), + application:unset_env(rabbit, log), + application:unset_env(lager, handlers), + application:unset_env(lager, extra_sinks), + ok = rabbit:start(), + timer:sleep(200), + rabbit_log:info("test info"), + + %% rotate logs when logging is turned off + ok = rabbit:stop(), + ok = clean_logs([LogFile], Suffix), + ok = application:set_env(rabbit, lager_default_file, false), + application:unset_env(rabbit, log), + application:unset_env(lager, handlers), + application:unset_env(lager, extra_sinks), + ok = rabbit:start(), + timer:sleep(200), + rabbit_log:error("test error"), + timer:sleep(200), + ?assertEqual([{error,enoent}], empty_files([LogFile])), + + %% cleanup + ok = rabbit:stop(), + ok = clean_logs([LogFile], Suffix), + ok = application:set_env(rabbit, lager_default_file, LogFile), + application:unset_env(rabbit, log), + application:unset_env(lager, handlers), + application:unset_env(lager, extra_sinks), + ok = rabbit:start(), + ok = test_logs_working([LogFile]), + passed. + +log_file_initialised_during_startup(Config) -> + passed = rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, log_file_initialised_during_startup1, [Config]). + +log_file_initialised_during_startup1(_Config) -> + [LogFile|_] = rabbit:log_locations(), + Suffix = ".0", + + %% start application with simple tty logging + ok = rabbit:stop(), + ok = clean_logs([LogFile], Suffix), + ok = application:set_env(rabbit, lager_default_file, tty), + application:unset_env(rabbit, log), + application:unset_env(lager, handlers), + application:unset_env(lager, extra_sinks), + ok = rabbit:start(), + + %% start application with logging to non-existing directory + NonExistent = rabbit_misc:format( + "/tmp/non-existent/~s.log", [?FUNCTION_NAME]), + delete_file(NonExistent), + delete_file(filename:dirname(NonExistent)), + ok = rabbit:stop(), + ct:pal("Setting lager_default_file to \"~s\"", [NonExistent]), + ok = application:set_env(rabbit, lager_default_file, NonExistent), + application:unset_env(rabbit, log), + application:unset_env(lager, handlers), + application:unset_env(lager, extra_sinks), + ok = rabbit:start(), + + %% clean up + ok = application:set_env(rabbit, lager_default_file, LogFile), + application:unset_env(rabbit, log), + application:unset_env(lager, handlers), + application:unset_env(lager, extra_sinks), + ok = rabbit:start(), + passed. + + +log_file_fails_to_initialise_during_startup(Config) -> + NonWritableDir = case os:type() of + {win32, _} -> "C:/Windows"; + _ -> "/" + end, + case file:open(filename:join(NonWritableDir, "test.log"), [write]) of + {error, eacces} -> + passed = rabbit_ct_broker_helpers:rpc( + Config, 0, + ?MODULE, log_file_fails_to_initialise_during_startup1, + [Config, NonWritableDir]); + %% macOS, "read only volume" + {error, erofs} -> + passed = rabbit_ct_broker_helpers:rpc( + Config, 0, + ?MODULE, log_file_fails_to_initialise_during_startup1, + [Config, NonWritableDir]); + {ok, Fd} -> + %% If the supposedly non-writable directory is writable + %% (e.g. we are running the testsuite on Windows as + %% Administrator), we skip this test. + file:close(Fd), + {skip, "Supposedly non-writable directory is writable"} + end. + +log_file_fails_to_initialise_during_startup1(_Config, NonWritableDir) -> + [LogFile|_] = rabbit:log_locations(), + delete_file(LogFile), + Fn = rabbit_misc:format("~s.log", [?FUNCTION_NAME]), + + %% start application with logging to directory with no + %% write permissions + NoPermission1 = filename:join(NonWritableDir, Fn), + delete_file(NoPermission1), + delete_file(filename:dirname(NoPermission1)), + + ok = rabbit:stop(), + ct:pal("Setting lager_default_file to \"~s\"", [NoPermission1]), + ok = application:set_env(rabbit, lager_default_file, NoPermission1), + application:unset_env(rabbit, log), + application:unset_env(lager, handlers), + application:unset_env(lager, extra_sinks), + + ct:pal("`rabbit` application env.: ~p", [application:get_all_env(rabbit)]), + ?assertThrow( + {error, {rabbit, {{cannot_log_to_file, _, _}, _}}}, + rabbit:start()), + + %% start application with logging to a subdirectory which + %% parent directory has no write permissions + NoPermission2 = filename:join([NonWritableDir, + "non-existent", + Fn]), + delete_file(NoPermission2), + delete_file(filename:dirname(NoPermission2)), + + ct:pal("Setting lager_default_file to \"~s\"", [NoPermission2]), + ok = application:set_env(rabbit, lager_default_file, NoPermission2), + application:unset_env(rabbit, log), + application:unset_env(lager, handlers), + application:unset_env(lager, extra_sinks), + + ct:pal("`rabbit` application env.: ~p", [application:get_all_env(rabbit)]), + ?assertThrow( + {error, {rabbit, {{cannot_log_to_file, _, _}, _}}}, + rabbit:start()), + + %% clean up + ok = application:set_env(rabbit, lager_default_file, LogFile), + application:unset_env(rabbit, log), + application:unset_env(lager, handlers), + application:unset_env(lager, extra_sinks), + ok = rabbit:start(), + passed. + +externally_rotated_logs_are_automatically_reopened(Config) -> + passed = rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, externally_rotated_logs_are_automatically_reopened1, [Config]). + +externally_rotated_logs_are_automatically_reopened1(_Config) -> + [LogFile|_] = rabbit:log_locations(), + + %% Make sure log file is opened + ok = test_logs_working([LogFile]), + + %% Move it away - i.e. external log rotation happened + file:rename(LogFile, [LogFile, ".rotation_test"]), + + %% New files should be created - test_logs_working/1 will check that + %% LogFile is not empty after doing some logging. And it's exactly + %% what we need to check here. + ok = test_logs_working([LogFile]), + passed. + +empty_or_nonexist_files(Files) -> + [case file:read_file_info(File) of + {ok, FInfo} -> FInfo#file_info.size == 0; + {error, enoent} -> true; + Error -> Error + end || File <- Files]. + +empty_files(Files) -> + [case file:read_file_info(File) of + {ok, FInfo} -> FInfo#file_info.size == 0; + Error -> Error + end || File <- Files]. + +non_empty_files(Files) -> + [case EmptyFile of + {error, Reason} -> {error, Reason}; + _ -> not(EmptyFile) + end || EmptyFile <- empty_files(Files)]. + +test_logs_working(LogFiles) -> + ok = rabbit_log:error("Log a test message"), + %% give the error loggers some time to catch up + timer:sleep(1000), + lists:all(fun(LogFile) -> [true] =:= non_empty_files([LogFile]) end, LogFiles), + ok. + +set_permissions(Path, Mode) -> + case file:read_file_info(Path) of + {ok, FInfo} -> file:write_file_info( + Path, + FInfo#file_info{mode=Mode}); + Error -> Error + end. + +clean_logs(Files, Suffix) -> + [begin + ok = delete_file(File), + ok = delete_file([File, Suffix]) + end || File <- Files], + ok. + +delete_file(File) -> + case file:delete(File) of + ok -> ok; + {error, enoent} -> ok; + Error -> Error + end. + +make_files_writable(Files) -> + [ok = file:write_file_info(File, #file_info{mode=8#644}) || + File <- Files], + ok. + +make_files_non_writable(Files) -> + [ok = file:write_file_info(File, #file_info{mode=8#444}) || + File <- Files], + ok. + +add_log_handlers(Handlers) -> + [ok = error_logger:add_report_handler(Handler, Args) || + {Handler, Args} <- Handlers], + ok. + +%% sasl_report_file_h returns [] during terminate +%% see: https://github.com/erlang/otp/blob/maint/lib/stdlib/src/error_logger_file_h.erl#L98 +%% +%% error_logger_file_h returns ok since OTP 18.1 +%% see: https://github.com/erlang/otp/blob/maint/lib/stdlib/src/error_logger_file_h.erl#L98 +delete_log_handlers(Handlers) -> + [ok_or_empty_list(error_logger:delete_report_handler(Handler)) + || Handler <- Handlers], + ok. + +ok_or_empty_list([]) -> + []; +ok_or_empty_list(ok) -> + ok. diff --git a/deps/rabbit/test/unit_operator_policy_SUITE.erl b/deps/rabbit/test/unit_operator_policy_SUITE.erl new file mode 100644 index 0000000000..ae3285bb55 --- /dev/null +++ b/deps/rabbit/test/unit_operator_policy_SUITE.erl @@ -0,0 +1,107 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2011-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(unit_operator_policy_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("rabbit_common/include/rabbit.hrl"). + +-compile(export_all). + +all() -> + [ + {group, parallel_tests} + ]. + +groups() -> + [ + {parallel_tests, [parallel], [ + merge_operator_policy_definitions + ]} + ]. + +init_per_testcase(_Testcase, Config) -> + Config. + +end_per_testcase(_TC, _Config) -> + ok. + + +%% ------------------------------------------------------------------- +%% Test Cases +%% ------------------------------------------------------------------- + +merge_operator_policy_definitions(_Config) -> + P1 = undefined, + P2 = [{definition, [{<<"message-ttl">>, 3000}]}], + ?assertEqual([{<<"message-ttl">>, 3000}], rabbit_policy:merge_operator_definitions(P1, P2)), + ?assertEqual([{<<"message-ttl">>, 3000}], rabbit_policy:merge_operator_definitions(P2, P1)), + + ?assertEqual([{<<"message-ttl">>, 3000}], rabbit_policy:merge_operator_definitions(P1, rabbit_data_coercion:to_map(P2))), + ?assertEqual([{<<"message-ttl">>, 3000}], rabbit_policy:merge_operator_definitions(rabbit_data_coercion:to_map(P2), P1)), + + ?assertEqual(undefined, rabbit_policy:merge_operator_definitions(undefined, undefined)), + + ?assertEqual([], rabbit_policy:merge_operator_definitions([], [])), + ?assertEqual([], rabbit_policy:merge_operator_definitions(#{}, [])), + ?assertEqual([], rabbit_policy:merge_operator_definitions(#{}, #{})), + ?assertEqual([], rabbit_policy:merge_operator_definitions([], #{})), + + %% operator policy takes precedence + ?assertEqual([{<<"message-ttl">>, 3000}], rabbit_policy:merge_operator_definitions( + [{definition, [ + {<<"message-ttl">>, 5000} + ]}], + [{definition, [ + {<<"message-ttl">>, 3000} + ]}] + )), + + ?assertEqual([{<<"delivery-limit">>, 20}, + {<<"message-ttl">>, 3000}], + rabbit_policy:merge_operator_definitions( + [{definition, [ + {<<"message-ttl">>, 5000}, + {<<"delivery-limit">>, 20} + ]}], + [{definition, [ + {<<"message-ttl">>, 3000} + ]}]) + ), + + ?assertEqual( + [{<<"delivery-limit">>, 20}, + {<<"message-ttl">>, 3000}, + {<<"unknown">>, <<"value">>}], + + rabbit_policy:merge_operator_definitions( + #{definition => #{ + <<"message-ttl">> => 5000, + <<"delivery-limit">> => 20 + }}, + #{definition => #{ + <<"message-ttl">> => 3000, + <<"unknown">> => <<"value">> + }}) + ), + + ?assertEqual( + [{<<"delivery-limit">>, 20}, + {<<"message-ttl">>, 3000}], + + rabbit_policy:merge_operator_definitions( + #{definition => #{ + <<"message-ttl">> => 5000, + <<"delivery-limit">> => 20 + }}, + [{definition, [ + {<<"message-ttl">>, 3000} + ]}]) + ), + + passed. diff --git a/deps/rabbit/test/unit_pg_local_SUITE.erl b/deps/rabbit/test/unit_pg_local_SUITE.erl new file mode 100644 index 0000000000..54fafdd340 --- /dev/null +++ b/deps/rabbit/test/unit_pg_local_SUITE.erl @@ -0,0 +1,103 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2011-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(unit_pg_local_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-compile(export_all). + +all() -> + [ + {group, sequential_tests} + ]. + +groups() -> + [ + {sequential_tests, [], [ + pg_local, + pg_local_with_unexpected_deaths1, + pg_local_with_unexpected_deaths2 + ]} + ]. + + +pg_local(_Config) -> + [P, Q] = [spawn(fun () -> receive X -> X end end) || _ <- lists:seq(0, 1)], + check_pg_local(ok, [], []), + %% P joins group a, then b, then a again + check_pg_local(pg_local:join(a, P), [P], []), + check_pg_local(pg_local:join(b, P), [P], [P]), + check_pg_local(pg_local:join(a, P), [P, P], [P]), + %% Q joins group a, then b, then b again + check_pg_local(pg_local:join(a, Q), [P, P, Q], [P]), + check_pg_local(pg_local:join(b, Q), [P, P, Q], [P, Q]), + check_pg_local(pg_local:join(b, Q), [P, P, Q], [P, Q, Q]), + %% P leaves groups a and a + check_pg_local(pg_local:leave(a, P), [P, Q], [P, Q, Q]), + check_pg_local(pg_local:leave(b, P), [P, Q], [Q, Q]), + %% leave/2 is idempotent + check_pg_local(pg_local:leave(a, P), [Q], [Q, Q]), + check_pg_local(pg_local:leave(a, P), [Q], [Q, Q]), + %% clean up all processes + [begin X ! done, + Ref = erlang:monitor(process, X), + receive {'DOWN', Ref, process, X, _Info} -> ok end + end || X <- [P, Q]], + %% ensure the groups are empty + check_pg_local(ok, [], []), + passed. + +pg_local_with_unexpected_deaths1(_Config) -> + [A, B] = [spawn(fun () -> receive X -> X end end) || _ <- lists:seq(0, 1)], + check_pg_local(ok, [], []), + %% A joins groups a and b + check_pg_local(pg_local:join(a, A), [A], []), + check_pg_local(pg_local:join(b, A), [A], [A]), + %% B joins group b + check_pg_local(pg_local:join(b, B), [A], [A, B]), + + [begin erlang:exit(X, sleep_now_in_a_fire), + Ref = erlang:monitor(process, X), + receive {'DOWN', Ref, process, X, _Info} -> ok end + end || X <- [A, B]], + %% ensure the groups are empty + check_pg_local(ok, [], []), + ?assertNot(erlang:is_process_alive(A)), + ?assertNot(erlang:is_process_alive(B)), + + passed. + +pg_local_with_unexpected_deaths2(_Config) -> + [A, B] = [spawn(fun () -> receive X -> X end end) || _ <- lists:seq(0, 1)], + check_pg_local(ok, [], []), + %% A joins groups a and b + check_pg_local(pg_local:join(a, A), [A], []), + check_pg_local(pg_local:join(b, A), [A], [A]), + %% B joins group b + check_pg_local(pg_local:join(b, B), [A], [A, B]), + + %% something else yanks a record (or all of them) from the pg_local + %% bookkeeping table + ok = pg_local:clear(), + + [begin erlang:exit(X, sleep_now_in_a_fire), + Ref = erlang:monitor(process, X), + receive {'DOWN', Ref, process, X, _Info} -> ok end + end || X <- [A, B]], + %% ensure the groups are empty + check_pg_local(ok, [], []), + ?assertNot(erlang:is_process_alive(A)), + ?assertNot(erlang:is_process_alive(B)), + + passed. + +check_pg_local(ok, APids, BPids) -> + ok = pg_local:sync(), + ?assertEqual([true, true], [lists:sort(Pids) == lists:sort(pg_local:get_members(Key)) || + {Key, Pids} <- [{a, APids}, {b, BPids}]]). diff --git a/deps/rabbit/test/unit_plugin_directories_SUITE.erl b/deps/rabbit/test/unit_plugin_directories_SUITE.erl new file mode 100644 index 0000000000..1195434fae --- /dev/null +++ b/deps/rabbit/test/unit_plugin_directories_SUITE.erl @@ -0,0 +1,76 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2011-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(unit_plugin_directories_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("rabbit_common/include/rabbit.hrl"). + +-compile(export_all). + +all() -> + [ + {group, parallel_tests} + ]. + +groups() -> + [ + {parallel_tests, [parallel], [ + listing_plugins_from_multiple_directories + ]} + ]. + + +%% ------------------------------------------------------------------- +%% Test Cases +%% ------------------------------------------------------------------- + +listing_plugins_from_multiple_directories(Config) -> + %% Generate some fake plugins in .ez files + FirstDir = filename:join([?config(priv_dir, Config), "listing_plugins_from_multiple_directories-1"]), + SecondDir = filename:join([?config(priv_dir, Config), "listing_plugins_from_multiple_directories-2"]), + ok = file:make_dir(FirstDir), + ok = file:make_dir(SecondDir), + lists:foreach(fun({Dir, AppName, Vsn}) -> + EzName = filename:join([Dir, io_lib:format("~s-~s.ez", [AppName, Vsn])]), + AppFileName = lists:flatten(io_lib:format("~s-~s/ebin/~s.app", [AppName, Vsn, AppName])), + AppFileContents = list_to_binary( + io_lib:format( + "~p.", + [{application, AppName, + [{vsn, Vsn}, + {applications, [kernel, stdlib, rabbit]}]}])), + {ok, {_, EzData}} = zip:zip(EzName, [{AppFileName, AppFileContents}], [memory]), + ok = file:write_file(EzName, EzData) + end, + [{FirstDir, plugin_first_dir, "3"}, + {SecondDir, plugin_second_dir, "4"}, + {FirstDir, plugin_both, "1"}, + {SecondDir, plugin_both, "2"}]), + + %% Everything was collected from both directories, plugin with higher + %% version should take precedence + PathSep = case os:type() of + {win32, _} -> ";"; + _ -> ":" + end, + Path = FirstDir ++ PathSep ++ SecondDir, + Got = lists:sort([{Name, Vsn} || #plugin{name = Name, version = Vsn} <- rabbit_plugins:list(Path)]), + %% `rabbit` was loaded automatically by `rabbit_plugins:list/1`. + %% We want to unload it now so it does not interfere with other + %% testcases. + application:unload(rabbit), + Expected = [{plugin_both, "2"}, {plugin_first_dir, "3"}, {plugin_second_dir, "4"}], + case Got of + Expected -> + ok; + _ -> + ct:pal("Got ~p~nExpected: ~p", [Got, Expected]), + exit({wrong_plugins_list, Got}) + end, + ok. diff --git a/deps/rabbit/test/unit_plugin_versioning_SUITE.erl b/deps/rabbit/test/unit_plugin_versioning_SUITE.erl new file mode 100644 index 0000000000..8032becedd --- /dev/null +++ b/deps/rabbit/test/unit_plugin_versioning_SUITE.erl @@ -0,0 +1,170 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2016-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(unit_plugin_versioning_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). + +-compile(export_all). + +all() -> + [ + {group, parallel_tests} + ]. + +groups() -> + [ + {parallel_tests, [parallel], [ + version_support, + plugin_validation + ]} + ]. + +%% ------------------------------------------------------------------- +%% Testsuite setup/teardown. +%% ------------------------------------------------------------------- + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(_, Config) -> + Config. + +end_per_group(_, Config) -> + Config. + +init_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_started(Config, Testcase). + +end_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_finished(Config, Testcase). + +%% ------------------------------------------------------------------- +%% Testcases. +%% ------------------------------------------------------------------- + +version_support(_Config) -> + Examples = [ + {[], "any version", true} %% anything goes + ,{[], "0.0.0", true} %% ditto + ,{[], "3.5.6", true} %% ditto + ,{["something"], "something", true} %% equal values match + ,{["3.5.4"], "something", false} + ,{["3.4.5", "3.6.0"], "0.0.0", true} %% zero version always match + ,{["3.4.5", "3.6.0"], "", true} %% empty version always match + ,{["something", "3.5.6"], "3.5.7", true} %% 3.5.7 matches ~> 3.5.6 + ,{["3.4.0", "3.5.6"], "3.6.1", false} %% 3.6.x isn't supported + ,{["3.5.2", "3.6.1", "3.7.1"], "3.5.2", true} %% 3.5.2 matches ~> 3.5.2 + ,{["3.5.2", "3.6.1", "3.7.1"], "3.5.1", false} %% lesser than the lower boundary + ,{["3.5.2", "3.6.1", "3.7.1"], "3.6.2", true} %% 3.6.2 matches ~> 3.6.1 + ,{["3.5.2", "3.6.1", "3.6.8"], "3.6.2", true} %% 3.6.2 still matches ~> 3.6.1 + ,{["3.5", "3.6", "3.7"], "3.5.1", true} %% x.y values equal to x.y.0 + ,{["3"], "3.5.1", false} %% x values are not supported + ,{["3.5.2", "3.6.1"], "3.6.2.999", true} %% x.y.z.p values are supported + ,{["3.5.2", "3.6.2.333"], "3.6.2.999", true} %% x.y.z.p values are supported + ,{["3.5.2", "3.6.2.333"], "3.6.2.222", false} %% x.y.z.p values are supported + ,{["3.6.0", "3.7.0"], "3.6.3-alpha.1", true} %% Pre-release versions handled like semver part + ,{["3.6.0", "3.7.0"], "3.7.0-alpha.89", true} + ], + + lists:foreach( + fun({Versions, RabbitVersion, Expected}) -> + {Expected, RabbitVersion, Versions} = + {rabbit_plugins:is_version_supported(RabbitVersion, Versions), + RabbitVersion, Versions} + end, + Examples), + ok. + +-record(validation_example, {rabbit_version, plugins, errors, valid}). + +plugin_validation(_Config) -> + Examples = [ + #validation_example{ + rabbit_version = "3.7.1", + plugins = + [{plugin_a, "3.7.2", ["3.5.6", "3.7.1"], []}, + {plugin_b, "3.7.2", ["3.7.0"], [{plugin_a, ["3.6.3", "3.7.1"]}]}], + errors = [], + valid = [plugin_a, plugin_b]}, + + #validation_example{ + rabbit_version = "3.7.1", + plugins = + [{plugin_a, "3.7.1", ["3.7.6"], []}, + {plugin_b, "3.7.2", ["3.7.0"], [{plugin_a, ["3.6.3", "3.7.0"]}]}], + errors = + [{plugin_a, [{broker_version_mismatch, "3.7.1", ["3.7.6"]}]}, + {plugin_b, [{missing_dependency, plugin_a}]}], + valid = [] + }, + + #validation_example{ + rabbit_version = "3.7.1", + plugins = + [{plugin_a, "3.7.1", ["3.7.6"], []}, + {plugin_b, "3.7.2", ["3.7.0"], [{plugin_a, ["3.7.0"]}]}, + {plugin_c, "3.7.2", ["3.7.0"], [{plugin_b, ["3.7.3"]}]}], + errors = + [{plugin_a, [{broker_version_mismatch, "3.7.1", ["3.7.6"]}]}, + {plugin_b, [{missing_dependency, plugin_a}]}, + {plugin_c, [{missing_dependency, plugin_b}]}], + valid = [] + }, + + #validation_example{ + rabbit_version = "3.7.1", + plugins = + [{plugin_a, "3.7.1", ["3.7.1"], []}, + {plugin_b, "3.7.2", ["3.7.0"], [{plugin_a, ["3.7.3"]}]}, + {plugin_d, "3.7.2", ["3.7.0"], [{plugin_c, ["3.7.3"]}]}], + errors = + [{plugin_b, [{{dependency_version_mismatch, "3.7.1", ["3.7.3"]}, plugin_a}]}, + {plugin_d, [{missing_dependency, plugin_c}]}], + valid = [plugin_a] + }, + #validation_example{ + rabbit_version = "0.0.0", + plugins = + [{plugin_a, "", ["3.7.1"], []}, + {plugin_b, "3.7.2", ["3.7.0"], [{plugin_a, ["3.7.3"]}]}], + errors = [], + valid = [plugin_a, plugin_b] + }], + lists:foreach( + fun(#validation_example{rabbit_version = RabbitVersion, + plugins = PluginsExamples, + errors = Errors, + valid = ExpectedValid}) -> + Plugins = make_plugins(PluginsExamples), + {Valid, Invalid} = rabbit_plugins:validate_plugins(Plugins, + RabbitVersion), + Errors = lists:reverse(Invalid), + ExpectedValid = lists:reverse(lists:map(fun(#plugin{name = Name}) -> + Name + end, + Valid)) + end, + Examples), + ok. + +make_plugins(Plugins) -> + lists:map( + fun({Name, Version, RabbitVersions, PluginsVersions}) -> + Deps = [K || {K,_V} <- PluginsVersions], + #plugin{name = Name, + version = Version, + dependencies = Deps, + broker_version_requirements = RabbitVersions, + dependency_version_requirements = PluginsVersions} + end, + Plugins). diff --git a/deps/rabbit/test/unit_policy_validators_SUITE.erl b/deps/rabbit/test/unit_policy_validators_SUITE.erl new file mode 100644 index 0000000000..c340d172af --- /dev/null +++ b/deps/rabbit/test/unit_policy_validators_SUITE.erl @@ -0,0 +1,207 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2011-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(unit_policy_validators_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-compile(export_all). + +all() -> + [ + {group, core_validators}, + {group, classic_queue_mirroring_validators} + ]. + +groups() -> + [ + {core_validators, [parallel], [ + alternate_exchange, + dead_letter_exchange, + dead_letter_routing_key, + message_ttl, + expires, + max_length, + max_length_bytes, + max_in_memory_length, + delivery_limit, + classic_queue_lazy_mode, + length_limit_overflow_mode + ]}, + + {classic_queue_mirroring_validators, [parallel], [ + classic_queue_ha_mode, + classic_queue_ha_params + ]} + ]. + +%% ------------------------------------------------------------------- +%% Test suite setup/teardown. +%% ------------------------------------------------------------------- + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(Group = classic_queue_mirroring_validators, Config) -> + Config1 = rabbit_ct_helpers:set_config(Config, [ + {rmq_nodename_suffix, Group}, + {rmq_nodes_count, 1} + ]), + rabbit_ct_helpers:run_steps(Config1, + rabbit_ct_broker_helpers:setup_steps()); +init_per_group(_, Config) -> + Config. + +end_per_group(classic_queue_mirroring_validators, Config) -> + rabbit_ct_helpers:run_steps(Config, + rabbit_ct_broker_helpers:teardown_steps()); +end_per_group(_, Config) -> + Config. + +init_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_started(Config, Testcase). + +end_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_finished(Config, Testcase). + +%% ------------------------------------------------------------------- +%% Core Validators +%% ------------------------------------------------------------------- + +alternate_exchange(_Config) -> + requires_binary_value(<<"alternate-exchange">>). + +dead_letter_exchange(_Config) -> + requires_binary_value(<<"dead-letter-exchange">>). + +dead_letter_routing_key(_Config) -> + requires_binary_value(<<"dead-letter-routing-key">>). + +message_ttl(_Config) -> + requires_non_negative_integer_value(<<"message-ttl">>). + +expires(_Config) -> + requires_positive_integer_value(<<"expires">>). + +max_length(_Config) -> + requires_non_negative_integer_value(<<"max-length">>). + +max_length_bytes(_Config) -> + requires_non_negative_integer_value(<<"max-length-bytes">>). + +max_in_memory_length(_Config) -> + requires_non_negative_integer_value(<<"max-in-memory-bytes">>). + +delivery_limit(_Config) -> + requires_non_negative_integer_value(<<"delivery-limit">>). + +classic_queue_lazy_mode(_Config) -> + test_valid_and_invalid_values(<<"queue-mode">>, + %% valid values + [<<"default">>, <<"lazy">>], + %% invalid values + [<<"unknown">>, <<"queue">>, <<"mode">>]). + +length_limit_overflow_mode(_Config) -> + test_valid_and_invalid_values(<<"overflow">>, + %% valid values + [<<"drop-head">>, <<"reject-publish">>, <<"reject-publish-dlx">>], + %% invalid values + [<<"unknown">>, <<"publish">>, <<"overflow">>, <<"mode">>]). + + +%% ------------------------------------------------------------------- +%% CMQ Validators +%% ------------------------------------------------------------------- + +classic_queue_ha_mode(Config) -> + rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, classic_queue_ha_mode1, [Config]). + +classic_queue_ha_mode1(_Config) -> + ?assertEqual(ok, rabbit_mirror_queue_misc:validate_policy([ + {<<"ha-mode">>, <<"exactly">>}, + {<<"ha-params">>, 2} + ])), + + ?assertEqual(ok, rabbit_mirror_queue_misc:validate_policy([ + {<<"ha-mode">>, <<"nodes">>}, + {<<"ha-params">>, [<<"rabbit@host1">>, <<"rabbit@host2">>]} + ])), + + ?assertEqual(ok, rabbit_mirror_queue_misc:validate_policy([ + {<<"ha-mode">>, <<"all">>} + ])), + + ?assertMatch({error, _, _}, rabbit_mirror_queue_misc:validate_policy([ + {<<"ha-mode">>, <<"lolwut">>}, + {<<"ha-params">>, 2} + ])). + +classic_queue_ha_params(Config) -> + rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, classic_queue_ha_mode1, [Config]). + +classic_queue_ha_params1(_Config) -> + ?assertMatch({error, _, _}, rabbit_mirror_queue_misc:validate_policy([ + {<<"ha-mode">>, <<"exactly">>}, + {<<"ha-params">>, <<"2">>} + ])), + + ?assertEqual(ok, rabbit_mirror_queue_misc:validate_policy([ + {<<"ha-mode">>, <<"nodes">>}, + {<<"ha-params">>, <<"lolwut">>} + ])), + + ?assertEqual(ok, rabbit_mirror_queue_misc:validate_policy([ + {<<"ha-mode">>, <<"all">>}, + {<<"ha-params">>, <<"lolwut">>} + ])), + + ?assertMatch({error, _, _}, rabbit_mirror_queue_misc:validate_policy([ + {<<"ha-mode">>, <<"lolwut">>}, + {<<"ha-params">>, 2} + ])). + +%% +%% Implementation +%% + +test_valid_and_invalid_values(Mod, Key, ValidValues, InvalidValues) -> + [begin + ?assertEqual(ok, Mod:validate_policy([ + {Key, Val} + ])) + end || Val <- ValidValues], + [begin + ?assertMatch({error, _, _}, Mod:validate_policy([ + {Key, Val} + ])) + end || Val <- InvalidValues]. + +test_valid_and_invalid_values(Key, ValidValues, InvalidValues) -> + test_valid_and_invalid_values(rabbit_policies, Key, ValidValues, InvalidValues). + +requires_binary_value(Key) -> + test_valid_and_invalid_values(Key, + [<<"a.binary">>, <<"b.binary">>], + [1, rabbit]). + +requires_positive_integer_value(Key) -> + test_valid_and_invalid_values(Key, + [1, 1000], + [0, -1, <<"a.binary">>]). + +requires_non_negative_integer_value(Key) -> + test_valid_and_invalid_values(Key, + [0, 1, 1000], + [-1000, -1, <<"a.binary">>]). diff --git a/deps/rabbit/test/unit_priority_queue_SUITE.erl b/deps/rabbit/test/unit_priority_queue_SUITE.erl new file mode 100644 index 0000000000..5587e7d61f --- /dev/null +++ b/deps/rabbit/test/unit_priority_queue_SUITE.erl @@ -0,0 +1,184 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2011-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(unit_priority_queue_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-compile(export_all). + +all() -> + [ + {group, sequential_tests} + ]. + +groups() -> + [ + {sequential_tests, [], [ + priority_queue + ]} + ]. + + +priority_queue(_Config) -> + + false = priority_queue:is_queue(not_a_queue), + + %% empty Q + Q = priority_queue:new(), + {true, true, 0, [], []} = test_priority_queue(Q), + + %% 1-4 element no-priority Q + true = lists:all(fun (X) -> X =:= passed end, + lists:map(fun test_simple_n_element_queue/1, + lists:seq(1, 4))), + + %% 1-element priority Q + Q1 = priority_queue:in(foo, 1, priority_queue:new()), + {true, false, 1, [{1, foo}], [foo]} = + test_priority_queue(Q1), + + %% 2-element same-priority Q + Q2 = priority_queue:in(bar, 1, Q1), + {true, false, 2, [{1, foo}, {1, bar}], [foo, bar]} = + test_priority_queue(Q2), + + %% 2-element different-priority Q + Q3 = priority_queue:in(bar, 2, Q1), + {true, false, 2, [{2, bar}, {1, foo}], [bar, foo]} = + test_priority_queue(Q3), + + %% 1-element negative priority Q + Q4 = priority_queue:in(foo, -1, priority_queue:new()), + {true, false, 1, [{-1, foo}], [foo]} = test_priority_queue(Q4), + + %% merge 2 * 1-element no-priority Qs + Q5 = priority_queue:join(priority_queue:in(foo, Q), + priority_queue:in(bar, Q)), + {true, false, 2, [{0, foo}, {0, bar}], [foo, bar]} = + test_priority_queue(Q5), + + %% merge 1-element no-priority Q with 1-element priority Q + Q6 = priority_queue:join(priority_queue:in(foo, Q), + priority_queue:in(bar, 1, Q)), + {true, false, 2, [{1, bar}, {0, foo}], [bar, foo]} = + test_priority_queue(Q6), + + %% merge 1-element priority Q with 1-element no-priority Q + Q7 = priority_queue:join(priority_queue:in(foo, 1, Q), + priority_queue:in(bar, Q)), + {true, false, 2, [{1, foo}, {0, bar}], [foo, bar]} = + test_priority_queue(Q7), + + %% merge 2 * 1-element same-priority Qs + Q8 = priority_queue:join(priority_queue:in(foo, 1, Q), + priority_queue:in(bar, 1, Q)), + {true, false, 2, [{1, foo}, {1, bar}], [foo, bar]} = + test_priority_queue(Q8), + + %% merge 2 * 1-element different-priority Qs + Q9 = priority_queue:join(priority_queue:in(foo, 1, Q), + priority_queue:in(bar, 2, Q)), + {true, false, 2, [{2, bar}, {1, foo}], [bar, foo]} = + test_priority_queue(Q9), + + %% merge 2 * 1-element different-priority Qs (other way around) + Q10 = priority_queue:join(priority_queue:in(bar, 2, Q), + priority_queue:in(foo, 1, Q)), + {true, false, 2, [{2, bar}, {1, foo}], [bar, foo]} = + test_priority_queue(Q10), + + %% merge 2 * 2-element multi-different-priority Qs + Q11 = priority_queue:join(Q6, Q5), + {true, false, 4, [{1, bar}, {0, foo}, {0, foo}, {0, bar}], + [bar, foo, foo, bar]} = test_priority_queue(Q11), + + %% and the other way around + Q12 = priority_queue:join(Q5, Q6), + {true, false, 4, [{1, bar}, {0, foo}, {0, bar}, {0, foo}], + [bar, foo, bar, foo]} = test_priority_queue(Q12), + + %% merge with negative priorities + Q13 = priority_queue:join(Q4, Q5), + {true, false, 3, [{0, foo}, {0, bar}, {-1, foo}], [foo, bar, foo]} = + test_priority_queue(Q13), + + %% and the other way around + Q14 = priority_queue:join(Q5, Q4), + {true, false, 3, [{0, foo}, {0, bar}, {-1, foo}], [foo, bar, foo]} = + test_priority_queue(Q14), + + %% joins with empty queues: + Q1 = priority_queue:join(Q, Q1), + Q1 = priority_queue:join(Q1, Q), + + %% insert with priority into non-empty zero-priority queue + Q15 = priority_queue:in(baz, 1, Q5), + {true, false, 3, [{1, baz}, {0, foo}, {0, bar}], [baz, foo, bar]} = + test_priority_queue(Q15), + + %% 1-element infinity priority Q + Q16 = priority_queue:in(foo, infinity, Q), + {true, false, 1, [{infinity, foo}], [foo]} = test_priority_queue(Q16), + + %% add infinity to 0-priority Q + Q17 = priority_queue:in(foo, infinity, priority_queue:in(bar, Q)), + {true, false, 2, [{infinity, foo}, {0, bar}], [foo, bar]} = + test_priority_queue(Q17), + + %% and the other way around + Q18 = priority_queue:in(bar, priority_queue:in(foo, infinity, Q)), + {true, false, 2, [{infinity, foo}, {0, bar}], [foo, bar]} = + test_priority_queue(Q18), + + %% add infinity to mixed-priority Q + Q19 = priority_queue:in(qux, infinity, Q3), + {true, false, 3, [{infinity, qux}, {2, bar}, {1, foo}], [qux, bar, foo]} = + test_priority_queue(Q19), + + %% merge the above with a negative priority Q + Q20 = priority_queue:join(Q19, Q4), + {true, false, 4, [{infinity, qux}, {2, bar}, {1, foo}, {-1, foo}], + [qux, bar, foo, foo]} = test_priority_queue(Q20), + + %% merge two infinity priority queues + Q21 = priority_queue:join(priority_queue:in(foo, infinity, Q), + priority_queue:in(bar, infinity, Q)), + {true, false, 2, [{infinity, foo}, {infinity, bar}], [foo, bar]} = + test_priority_queue(Q21), + + %% merge two mixed priority with infinity queues + Q22 = priority_queue:join(Q18, Q20), + {true, false, 6, [{infinity, foo}, {infinity, qux}, {2, bar}, {1, foo}, + {0, bar}, {-1, foo}], [foo, qux, bar, foo, bar, foo]} = + test_priority_queue(Q22), + + passed. + +priority_queue_in_all(Q, L) -> + lists:foldl(fun (X, Acc) -> priority_queue:in(X, Acc) end, Q, L). + +priority_queue_out_all(Q) -> + case priority_queue:out(Q) of + {empty, _} -> []; + {{value, V}, Q1} -> [V | priority_queue_out_all(Q1)] + end. + +test_priority_queue(Q) -> + {priority_queue:is_queue(Q), + priority_queue:is_empty(Q), + priority_queue:len(Q), + priority_queue:to_list(Q), + priority_queue_out_all(Q)}. + +test_simple_n_element_queue(N) -> + Items = lists:seq(1, N), + Q = priority_queue_in_all(priority_queue:new(), Items), + ToListRes = [{0, X} || X <- Items], + {true, false, N, ToListRes, Items} = test_priority_queue(Q), + passed. diff --git a/deps/rabbit/test/unit_queue_consumers_SUITE.erl b/deps/rabbit/test/unit_queue_consumers_SUITE.erl new file mode 100644 index 0000000000..0f48ea65b4 --- /dev/null +++ b/deps/rabbit/test/unit_queue_consumers_SUITE.erl @@ -0,0 +1,121 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2018-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(unit_queue_consumers_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-compile(export_all). + +all() -> + [ + is_same, + get_consumer, + get, + list_consumers + ]. + +is_same(_Config) -> + ?assertEqual( + true, + rabbit_queue_consumers:is_same( + self(), <<"1">>, + consumer(self(), <<"1">>) + )), + ?assertEqual( + false, + rabbit_queue_consumers:is_same( + self(), <<"1">>, + consumer(self(), <<"2">>) + )), + Pid = spawn(?MODULE, function_for_process, []), + Pid ! whatever, + ?assertEqual( + false, + rabbit_queue_consumers:is_same( + self(), <<"1">>, + consumer(Pid, <<"1">>) + )), + ok. + +get(_Config) -> + Pid = spawn(?MODULE, function_for_process, []), + Pid ! whatever, + State = state(consumers([consumer(self(), <<"1">>), consumer(Pid, <<"2">>), consumer(self(), <<"3">>)])), + {Pid, {consumer, <<"2">>, _, _, _, _}} = + rabbit_queue_consumers:get(Pid, <<"2">>, State), + ?assertEqual( + undefined, + rabbit_queue_consumers:get(self(), <<"2">>, State) + ), + ?assertEqual( + undefined, + rabbit_queue_consumers:get(Pid, <<"1">>, State) + ), + ok. + +get_consumer(_Config) -> + Pid = spawn(unit_queue_consumers_SUITE, function_for_process, []), + Pid ! whatever, + State = state(consumers([consumer(self(), <<"1">>), consumer(Pid, <<"2">>), consumer(self(), <<"3">>)])), + {_Pid, {consumer, _, _, _, _, _}} = + rabbit_queue_consumers:get_consumer(State), + ?assertEqual( + undefined, + rabbit_queue_consumers:get_consumer(state(consumers([]))) + ), + ok. + +list_consumers(_Config) -> + State = state(consumers([consumer(self(), <<"1">>), consumer(self(), <<"2">>), consumer(self(), <<"3">>)])), + Consumer = rabbit_queue_consumers:get_consumer(State), + {_Pid, ConsumerRecord} = Consumer, + CTag = rabbit_queue_consumers:consumer_tag(ConsumerRecord), + ConsumersWithSingleActive = rabbit_queue_consumers:all(State, Consumer, true), + ?assertEqual(3, length(ConsumersWithSingleActive)), + lists:foldl(fun({Pid, Tag, _, _, Active, ActivityStatus, _, _}, _Acc) -> + ?assertEqual(self(), Pid), + case Tag of + CTag -> + ?assert(Active), + ?assertEqual(single_active, ActivityStatus); + _ -> + ?assertNot(Active), + ?assertEqual(waiting, ActivityStatus) + end + end, [], ConsumersWithSingleActive), + ConsumersNoSingleActive = rabbit_queue_consumers:all(State, none, false), + ?assertEqual(3, length(ConsumersNoSingleActive)), + lists:foldl(fun({Pid, _, _, _, Active, ActivityStatus, _, _}, _Acc) -> + ?assertEqual(self(), Pid), + ?assert(Active), + ?assertEqual(up, ActivityStatus) + end, [], ConsumersNoSingleActive), + ok. + +consumers([]) -> + priority_queue:new(); +consumers(Consumers) -> + consumers(Consumers, priority_queue:new()). + +consumers([H], Q) -> + priority_queue:in(H, Q); +consumers([H | T], Q) -> + consumers(T, priority_queue:in(H, Q)). + + +consumer(Pid, ConsumerTag) -> + {Pid, {consumer, ConsumerTag, true, 1, [], <<"guest">>}}. + +state(Consumers) -> + {state, Consumers, {}}. + +function_for_process() -> + receive + _ -> ok + end. diff --git a/deps/rabbit/test/unit_stats_and_metrics_SUITE.erl b/deps/rabbit/test/unit_stats_and_metrics_SUITE.erl new file mode 100644 index 0000000000..2ffed514e1 --- /dev/null +++ b/deps/rabbit/test/unit_stats_and_metrics_SUITE.erl @@ -0,0 +1,266 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2011-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(unit_stats_and_metrics_SUITE). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("kernel/include/file.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). + +-compile(export_all). + +-define(TIMEOUT, 30000). + +all() -> + [ + {group, non_parallel_tests} + ]. + +groups() -> + [ + {non_parallel_tests, [], [ + channel_statistics, + head_message_timestamp_statistics + ]} + ]. + +%% ------------------------------------------------------------------- +%% Testsuite setup/teardown +%% ------------------------------------------------------------------- + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(Group, Config) -> + Config1 = rabbit_ct_helpers:set_config(Config, [ + {rmq_nodename_suffix, Group}, + {rmq_nodes_count, 1} + ]), + rabbit_ct_helpers:run_steps(Config1, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()). + +end_per_group(_Group, Config) -> + rabbit_ct_helpers:run_steps(Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()). + +init_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_started(Config, Testcase). + +end_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_finished(Config, Testcase). + + +%% ------------------------------------------------------------------- +%% Statistics. +%% ------------------------------------------------------------------- + +channel_statistics(Config) -> + passed = rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, channel_statistics1, [Config]). + +channel_statistics1(_Config) -> + application:set_env(rabbit, collect_statistics, fine), + + %% ATM this just tests the queue / exchange stats in channels. That's + %% by far the most complex code though. + + %% Set up a channel and queue + {_Writer, Ch} = test_spawn(), + rabbit_channel:do(Ch, #'queue.declare'{}), + QName = receive #'queue.declare_ok'{queue = Q0} -> Q0 + after ?TIMEOUT -> throw(failed_to_receive_queue_declare_ok) + end, + QRes = rabbit_misc:r(<<"/">>, queue, QName), + X = rabbit_misc:r(<<"/">>, exchange, <<"">>), + + dummy_event_receiver:start(self(), [node()], [channel_stats]), + + %% Check stats empty + Check1 = fun() -> + [] = ets:match(channel_queue_metrics, {Ch, QRes}), + [] = ets:match(channel_exchange_metrics, {Ch, X}), + [] = ets:match(channel_queue_exchange_metrics, + {Ch, {QRes, X}}) + end, + test_ch_metrics(Check1, ?TIMEOUT), + + %% Publish and get a message + rabbit_channel:do(Ch, #'basic.publish'{exchange = <<"">>, + routing_key = QName}, + rabbit_basic:build_content(#'P_basic'{}, <<"">>)), + rabbit_channel:do(Ch, #'basic.get'{queue = QName}), + + %% Check the stats reflect that + Check2 = fun() -> + [{{Ch, QRes}, 1, 0, 0, 0, 0, 0, 0, 0}] = ets:lookup( + channel_queue_metrics, + {Ch, QRes}), + [{{Ch, X}, 1, 0, 0, 0, 0}] = ets:lookup( + channel_exchange_metrics, + {Ch, X}), + [{{Ch, {QRes, X}}, 1, 0}] = ets:lookup( + channel_queue_exchange_metrics, + {Ch, {QRes, X}}) + end, + test_ch_metrics(Check2, ?TIMEOUT), + + %% Check the stats are marked for removal on queue deletion. + rabbit_channel:do(Ch, #'queue.delete'{queue = QName}), + Check3 = fun() -> + [{{Ch, QRes}, 1, 0, 0, 0, 0, 0, 0, 1}] = ets:lookup( + channel_queue_metrics, + {Ch, QRes}), + [{{Ch, X}, 1, 0, 0, 0, 0}] = ets:lookup( + channel_exchange_metrics, + {Ch, X}), + [{{Ch, {QRes, X}}, 1, 1}] = ets:lookup( + channel_queue_exchange_metrics, + {Ch, {QRes, X}}) + end, + test_ch_metrics(Check3, ?TIMEOUT), + + %% Check the garbage collection removes stuff. + force_metric_gc(), + Check4 = fun() -> + [] = ets:lookup(channel_queue_metrics, {Ch, QRes}), + [{{Ch, X}, 1, 0, 0, 0, 0}] = ets:lookup( + channel_exchange_metrics, + {Ch, X}), + [] = ets:lookup(channel_queue_exchange_metrics, + {Ch, {QRes, X}}) + end, + test_ch_metrics(Check4, ?TIMEOUT), + + rabbit_channel:shutdown(Ch), + dummy_event_receiver:stop(), + passed. + +force_metric_gc() -> + timer:sleep(300), + rabbit_core_metrics_gc ! start_gc, + gen_server:call(rabbit_core_metrics_gc, test). + +test_ch_metrics(Fun, Timeout) when Timeout =< 0 -> + Fun(); +test_ch_metrics(Fun, Timeout) -> + try + Fun() + catch + _:{badmatch, _} -> + timer:sleep(1000), + test_ch_metrics(Fun, Timeout - 1000) + end. + +head_message_timestamp_statistics(Config) -> + passed = rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, head_message_timestamp1, [Config]). + +head_message_timestamp1(_Config) -> + %% there is no convenient rabbit_channel API for confirms + %% this test could use, so it relies on tx.* methods + %% and gen_server2 flushing + application:set_env(rabbit, collect_statistics, fine), + + %% Set up a channel and queue + {_Writer, Ch} = test_spawn(), + rabbit_channel:do(Ch, #'queue.declare'{}), + QName = receive #'queue.declare_ok'{queue = Q0} -> Q0 + after ?TIMEOUT -> throw(failed_to_receive_queue_declare_ok) + end, + QRes = rabbit_misc:r(<<"/">>, queue, QName), + + {ok, Q1} = rabbit_amqqueue:lookup(QRes), + QPid = amqqueue:get_pid(Q1), + + %% Set up event receiver for queue + dummy_event_receiver:start(self(), [node()], [queue_stats]), + + %% the head timestamp field is empty when the queue is empty + test_queue_statistics_receive_event(QPid, + fun (E) -> + (proplists:get_value(name, E) == QRes) + and + (proplists:get_value(head_message_timestamp, E) == '') + end), + + rabbit_channel:do(Ch, #'tx.select'{}), + receive #'tx.select_ok'{} -> ok + after ?TIMEOUT -> throw(failed_to_receive_tx_select_ok) + end, + + %% Publish two messages and check that the timestamp is that of the first message + rabbit_channel:do(Ch, #'basic.publish'{exchange = <<"">>, + routing_key = QName}, + rabbit_basic:build_content(#'P_basic'{timestamp = 1}, <<"">>)), + rabbit_channel:do(Ch, #'basic.publish'{exchange = <<"">>, + routing_key = QName}, + rabbit_basic:build_content(#'P_basic'{timestamp = 2}, <<"">>)), + rabbit_channel:do(Ch, #'tx.commit'{}), + rabbit_channel:flush(Ch), + receive #'tx.commit_ok'{} -> ok + after ?TIMEOUT -> throw(failed_to_receive_tx_commit_ok) + end, + test_queue_statistics_receive_event(QPid, + fun (E) -> + (proplists:get_value(name, E) == QRes) + and + (proplists:get_value(head_message_timestamp, E) == 1) + end), + + %% Consume a message and check that the timestamp is now that of the second message + rabbit_channel:do(Ch, #'basic.get'{queue = QName, no_ack = true}), + test_queue_statistics_receive_event(QPid, + fun (E) -> + (proplists:get_value(name, E) == QRes) + and + (proplists:get_value(head_message_timestamp, E) == 2) + end), + + %% Consume one more message and check again + rabbit_channel:do(Ch, #'basic.get'{queue = QName, no_ack = true}), + test_queue_statistics_receive_event(QPid, + fun (E) -> + (proplists:get_value(name, E) == QRes) + and + (proplists:get_value(head_message_timestamp, E) == '') + end), + + %% Tear down + rabbit_channel:do(Ch, #'queue.delete'{queue = QName}), + rabbit_channel:shutdown(Ch), + dummy_event_receiver:stop(), + + passed. + +test_queue_statistics_receive_event(Q, Matcher) -> + %% Q ! emit_stats, + test_queue_statistics_receive_event1(Q, Matcher). + +test_queue_statistics_receive_event1(Q, Matcher) -> + receive #event{type = queue_stats, props = Props} -> + case Matcher(Props) of + true -> Props; + _ -> test_queue_statistics_receive_event1(Q, Matcher) + end + after ?TIMEOUT -> throw(failed_to_receive_event) + end. + +test_spawn() -> + {Writer, _Limiter, Ch} = rabbit_ct_broker_helpers:test_channel(), + ok = rabbit_channel:do(Ch, #'channel.open'{}), + receive #'channel.open_ok'{} -> ok + after ?TIMEOUT -> throw(failed_to_receive_channel_open_ok) + end, + {Writer, Ch}. diff --git a/deps/rabbit/test/unit_supervisor2_SUITE.erl b/deps/rabbit/test/unit_supervisor2_SUITE.erl new file mode 100644 index 0000000000..50633984e2 --- /dev/null +++ b/deps/rabbit/test/unit_supervisor2_SUITE.erl @@ -0,0 +1,69 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2011-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(unit_supervisor2_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-compile(export_all). + +all() -> + [ + {group, sequential_tests} + ]. + +groups() -> + [ + {sequential_tests, [], [ + check_shutdown_stop, + check_shutdown_ignored + ]} + ]. + +%% ------------------------------------------------------------------- +%% Test Cases +%% ------------------------------------------------------------------- + +check_shutdown_stop(_Config) -> + ok = check_shutdown(stop, 200, 200, 2000). + +check_shutdown_ignored(_Config) -> + ok = check_shutdown(ignored, 1, 2, 2000). + +check_shutdown(SigStop, Iterations, ChildCount, SupTimeout) -> + {ok, Sup} = supervisor2:start_link(dummy_supervisor2, [SupTimeout]), + Res = lists:foldl( + fun (I, ok) -> + TestSupPid = erlang:whereis(dummy_supervisor2), + ChildPids = + [begin + {ok, ChildPid} = + supervisor2:start_child(TestSupPid, []), + ChildPid + end || _ <- lists:seq(1, ChildCount)], + MRef = erlang:monitor(process, TestSupPid), + [P ! SigStop || P <- ChildPids], + ok = supervisor2:terminate_child(Sup, test_sup), + {ok, _} = supervisor2:restart_child(Sup, test_sup), + receive + {'DOWN', MRef, process, TestSupPid, shutdown} -> + ok; + {'DOWN', MRef, process, TestSupPid, Reason} -> + {error, {I, Reason}} + end; + (_, R) -> + R + end, ok, lists:seq(1, Iterations)), + unlink(Sup), + MSupRef = erlang:monitor(process, Sup), + exit(Sup, shutdown), + receive + {'DOWN', MSupRef, process, Sup, _Reason} -> + ok + end, + Res. diff --git a/deps/rabbit/test/unit_vm_memory_monitor_SUITE.erl b/deps/rabbit/test/unit_vm_memory_monitor_SUITE.erl new file mode 100644 index 0000000000..193df1f956 --- /dev/null +++ b/deps/rabbit/test/unit_vm_memory_monitor_SUITE.erl @@ -0,0 +1,95 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2011-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(unit_vm_memory_monitor_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-compile(export_all). + +all() -> + [ + {group, sequential_tests} + ]. + +groups() -> + [ + {sequential_tests, [], [ + parse_line_linux, + set_vm_memory_high_watermark_command + ]} + ]. + + +%% ------------------------------------------------------------------- +%% Testsuite setup/teardown +%% ------------------------------------------------------------------- + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(Group, Config) -> + Config1 = rabbit_ct_helpers:set_config(Config, [ + {rmq_nodename_suffix, Group}, + {rmq_nodes_count, 1} + ]), + rabbit_ct_helpers:run_steps(Config1, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()). + +end_per_group(_Group, Config) -> + rabbit_ct_helpers:run_steps(Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()). + +init_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_started(Config, Testcase). + +end_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_finished(Config, Testcase). + + +%% ------------------------------------------------------------------- +%% Test cases +%% ------------------------------------------------------------------- + + +parse_line_linux(_Config) -> + lists:foreach(fun ({S, {K, V}}) -> + {K, V} = vm_memory_monitor:parse_line_linux(S) + end, + [{"MemTotal: 0 kB", {'MemTotal', 0}}, + {"MemTotal: 502968 kB ", {'MemTotal', 515039232}}, + {"MemFree: 178232 kB", {'MemFree', 182509568}}, + {"MemTotal: 50296888", {'MemTotal', 50296888}}, + {"MemTotal 502968 kB", {'MemTotal', 515039232}}, + {"MemTotal 50296866 ", {'MemTotal', 50296866}}]), + ok. + +set_vm_memory_high_watermark_command(Config) -> + rabbit_ct_broker_helpers:rpc(Config, 0, + ?MODULE, set_vm_memory_high_watermark_command1, [Config]). + +set_vm_memory_high_watermark_command1(_Config) -> + MemLimitRatio = 1.0, + MemTotal = vm_memory_monitor:get_total_memory(), + + vm_memory_monitor:set_vm_memory_high_watermark(MemLimitRatio), + MemLimit = vm_memory_monitor:get_memory_limit(), + case MemLimit of + MemTotal -> ok; + _ -> MemTotalToMemLimitRatio = MemLimit * 100.0 / MemTotal / 100, + ct:fail( + "Expected memory high watermark to be ~p (~s), but it was ~p (~.1f)", + [MemTotal, MemLimitRatio, MemLimit, MemTotalToMemLimitRatio] + ) + end. diff --git a/deps/rabbit/test/upgrade_preparation_SUITE.erl b/deps/rabbit/test/upgrade_preparation_SUITE.erl new file mode 100644 index 0000000000..880238515a --- /dev/null +++ b/deps/rabbit/test/upgrade_preparation_SUITE.erl @@ -0,0 +1,109 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(upgrade_preparation_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). + +-compile(export_all). + +all() -> + [ + {group, clustered} + ]. + +groups() -> + [ + {clustered, [], [ + await_quorum_plus_one + ]} + ]. + + +%% ------------------------------------------------------------------- +%% Test Case +%% ------------------------------------------------------------------- + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(Group, Config) -> + Config1 = rabbit_ct_helpers:set_config(Config, [ + {rmq_nodes_count, 3}, + {rmq_nodename_suffix, Group} + ]), + rabbit_ct_helpers:run_steps(Config1, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()). + +end_per_group(_Group, Config) -> + rabbit_ct_helpers:run_steps(Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()). + + +init_per_testcase(TestCase, Config) -> + rabbit_ct_helpers:testcase_started(Config, TestCase), + case rabbit_ct_broker_helpers:enable_feature_flag(Config, quorum_queue) of + ok -> Config; + Skip -> Skip + end. + +end_per_testcase(TestCase, Config) -> + rabbit_ct_helpers:testcase_finished(Config, TestCase). + + + +%% +%% Test Cases +%% + +-define(WAITING_INTERVAL, 10000). + +await_quorum_plus_one(Config) -> + catch delete_queues(), + [A, B, _C] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + Ch = rabbit_ct_client_helpers:open_channel(Config, A), + declare(Ch, <<"qq.1">>, [{<<"x-queue-type">>, longstr, <<"quorum">>}]), + timer:sleep(100), + ?assert(await_quorum_plus_one(Config, 0)), + + ok = rabbit_ct_broker_helpers:stop_node(Config, B), + ?assertNot(await_quorum_plus_one(Config, 0)), + + ok = rabbit_ct_broker_helpers:start_node(Config, B), + ?assert(await_quorum_plus_one(Config, 0)). + +%% +%% Implementation +%% + +declare(Ch, Q) -> + declare(Ch, Q, []). + +declare(Ch, Q, Args) -> + amqp_channel:call(Ch, #'queue.declare'{queue = Q, + durable = true, + auto_delete = false, + arguments = Args}). + +delete_queues() -> + [rabbit_amqqueue:delete(Q, false, false, <<"tests">>) || Q <- rabbit_amqqueue:list()]. + +await_quorum_plus_one(Config, Node) -> + await_quorum_plus_one(Config, Node, ?WAITING_INTERVAL). + +await_quorum_plus_one(Config, Node, Timeout) -> + rabbit_ct_broker_helpers:rpc(Config, Node, + rabbit_upgrade_preparation, await_online_quorum_plus_one, [Timeout], Timeout + 500). + diff --git a/deps/rabbit/test/vhost_SUITE.erl b/deps/rabbit/test/vhost_SUITE.erl new file mode 100644 index 0000000000..4e6ffe0d74 --- /dev/null +++ b/deps/rabbit/test/vhost_SUITE.erl @@ -0,0 +1,381 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2011-2020 VMware, Inc. or its affiliates. All rights reserved. +%% + +-module(vhost_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-compile(export_all). + +all() -> + [ + {group, cluster_size_1_network}, + {group, cluster_size_2_network}, + {group, cluster_size_1_direct}, + {group, cluster_size_2_direct} + ]. + +groups() -> + ClusterSize1Tests = [ + single_node_vhost_deletion_forces_connection_closure, + vhost_failure_forces_connection_closure, + vhost_creation_idempotency + ], + ClusterSize2Tests = [ + cluster_vhost_deletion_forces_connection_closure, + vhost_failure_forces_connection_closure, + vhost_failure_forces_connection_closure_on_failure_node, + node_starts_with_dead_vhosts, + node_starts_with_dead_vhosts_with_mirrors, + vhost_creation_idempotency + ], + [ + {cluster_size_1_network, [], ClusterSize1Tests}, + {cluster_size_2_network, [], ClusterSize2Tests}, + {cluster_size_1_direct, [], ClusterSize1Tests}, + {cluster_size_2_direct, [], ClusterSize2Tests} + ]. + +suite() -> + [ + %% If a test hangs, no need to wait for 30 minutes. + {timetrap, {minutes, 8}} + ]. + +%% see partitions_SUITE +-define(DELAY, 9000). + +%% ------------------------------------------------------------------- +%% Testsuite setup/teardown. +%% ------------------------------------------------------------------- + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(cluster_size_1_network, Config) -> + Config1 = rabbit_ct_helpers:set_config(Config, [{connection_type, network}]), + init_per_multinode_group(cluster_size_1_network, Config1, 1); +init_per_group(cluster_size_2_network, Config) -> + Config1 = rabbit_ct_helpers:set_config(Config, [{connection_type, network}]), + init_per_multinode_group(cluster_size_2_network, Config1, 2); +init_per_group(cluster_size_1_direct, Config) -> + Config1 = rabbit_ct_helpers:set_config(Config, [{connection_type, direct}]), + init_per_multinode_group(cluster_size_1_direct, Config1, 1); +init_per_group(cluster_size_2_direct, Config) -> + Config1 = rabbit_ct_helpers:set_config(Config, [{connection_type, direct}]), + init_per_multinode_group(cluster_size_2_direct, Config1, 2). + +init_per_multinode_group(_Group, Config, NodeCount) -> + Suffix = rabbit_ct_helpers:testcase_absname(Config, "", "-"), + Config1 = rabbit_ct_helpers:set_config(Config, [ + {rmq_nodes_count, NodeCount}, + {rmq_nodename_suffix, Suffix} + ]), + + rabbit_ct_helpers:run_steps(Config1, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()). + +end_per_group(_Group, Config) -> + rabbit_ct_helpers:run_steps(Config, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()). + +init_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_started(Config, Testcase), + clear_all_connection_tracking_tables(Config), + Config. + +end_per_testcase(Testcase, Config) -> + VHost1 = <<"vhost1">>, + VHost2 = <<"vhost2">>, + case Testcase of + cluster_vhost_deletion_forces_connection_closure -> ok; + single_node_vhost_deletion_forces_connection_closure -> ok; + _ -> + delete_vhost(Config, VHost2) + end, + delete_vhost(Config, VHost1), + clear_all_connection_tracking_tables(Config), + rabbit_ct_helpers:testcase_finished(Config, Testcase). + +delete_vhost(Config, VHost) -> + case rabbit_ct_broker_helpers:delete_vhost(Config, VHost) of + ok -> ok; + {error, {no_such_vhost, _}} -> ok + end. + +clear_all_connection_tracking_tables(Config) -> + [rabbit_ct_broker_helpers:rpc(Config, + N, + rabbit_connection_tracking, + clear_tracked_connection_tables_for_this_node, + []) || N <- rabbit_ct_broker_helpers:get_node_configs(Config, nodename)]. + +%% ------------------------------------------------------------------- +%% Test cases. +%% ------------------------------------------------------------------- + +single_node_vhost_deletion_forces_connection_closure(Config) -> + VHost1 = <<"vhost1">>, + VHost2 = <<"vhost2">>, + + set_up_vhost(Config, VHost1), + set_up_vhost(Config, VHost2), + + ?assertEqual(0, count_connections_in(Config, VHost1)), + ?assertEqual(0, count_connections_in(Config, VHost2)), + + [Conn1] = open_connections(Config, [{0, VHost1}]), + ?assertEqual(1, count_connections_in(Config, VHost1)), + + [_Conn2] = open_connections(Config, [{0, VHost2}]), + ?assertEqual(1, count_connections_in(Config, VHost2)), + + rabbit_ct_broker_helpers:delete_vhost(Config, VHost2), + timer:sleep(200), + ?assertEqual(0, count_connections_in(Config, VHost2)), + + close_connections([Conn1]), + ?assertEqual(0, count_connections_in(Config, VHost1)). + +vhost_failure_forces_connection_closure(Config) -> + VHost1 = <<"vhost1">>, + VHost2 = <<"vhost2">>, + + set_up_vhost(Config, VHost1), + set_up_vhost(Config, VHost2), + + ?assertEqual(0, count_connections_in(Config, VHost1)), + ?assertEqual(0, count_connections_in(Config, VHost2)), + + [Conn1] = open_connections(Config, [{0, VHost1}]), + ?assertEqual(1, count_connections_in(Config, VHost1)), + + [_Conn2] = open_connections(Config, [{0, VHost2}]), + ?assertEqual(1, count_connections_in(Config, VHost2)), + + rabbit_ct_broker_helpers:force_vhost_failure(Config, VHost2), + timer:sleep(200), + ?assertEqual(0, count_connections_in(Config, VHost2)), + + close_connections([Conn1]), + ?assertEqual(0, count_connections_in(Config, VHost1)). + + +vhost_failure_forces_connection_closure_on_failure_node(Config) -> + VHost1 = <<"vhost1">>, + VHost2 = <<"vhost2">>, + + set_up_vhost(Config, VHost1), + set_up_vhost(Config, VHost2), + + ?assertEqual(0, count_connections_in(Config, VHost1)), + ?assertEqual(0, count_connections_in(Config, VHost2)), + + [Conn1] = open_connections(Config, [{0, VHost1}]), + ?assertEqual(1, count_connections_in(Config, VHost1)), + + [_Conn20] = open_connections(Config, [{0, VHost2}]), + [_Conn21] = open_connections(Config, [{1, VHost2}]), + ?assertEqual(2, count_connections_in(Config, VHost2)), + + rabbit_ct_broker_helpers:force_vhost_failure(Config, 0, VHost2), + timer:sleep(200), + %% Vhost2 connection on node 1 is still alive + ?assertEqual(1, count_connections_in(Config, VHost2)), + %% Vhost1 connection on node 0 is still alive + ?assertEqual(1, count_connections_in(Config, VHost1)), + + close_connections([Conn1]), + ?assertEqual(0, count_connections_in(Config, VHost1)). + + +cluster_vhost_deletion_forces_connection_closure(Config) -> + VHost1 = <<"vhost1">>, + VHost2 = <<"vhost2">>, + + set_up_vhost(Config, VHost1), + set_up_vhost(Config, VHost2), + + ?assertEqual(0, count_connections_in(Config, VHost1)), + ?assertEqual(0, count_connections_in(Config, VHost2)), + + [Conn1] = open_connections(Config, [{0, VHost1}]), + ?assertEqual(1, count_connections_in(Config, VHost1)), + + [_Conn2] = open_connections(Config, [{1, VHost2}]), + ?assertEqual(1, count_connections_in(Config, VHost2)), + + rabbit_ct_broker_helpers:delete_vhost(Config, VHost2), + timer:sleep(200), + ?assertEqual(0, count_connections_in(Config, VHost2)), + + close_connections([Conn1]), + ?assertEqual(0, count_connections_in(Config, VHost1)). + +node_starts_with_dead_vhosts(Config) -> + VHost1 = <<"vhost1">>, + VHost2 = <<"vhost2">>, + + set_up_vhost(Config, VHost1), + set_up_vhost(Config, VHost2), + + Conn = rabbit_ct_client_helpers:open_unmanaged_connection(Config, 1, VHost1), + {ok, Chan} = amqp_connection:open_channel(Conn), + + QName = <<"node_starts_with_dead_vhosts-q-1">>, + amqp_channel:call(Chan, #'queue.declare'{queue = QName, durable = true}), + rabbit_ct_client_helpers:publish(Chan, QName, 10), + + DataStore1 = rabbit_ct_broker_helpers:rpc( + Config, 1, rabbit_vhost, msg_store_dir_path, [VHost1]), + + rabbit_ct_broker_helpers:stop_node(Config, 1), + + file:write_file(filename:join(DataStore1, "recovery.dets"), <<"garbage">>), + + %% The node should start without a vhost + ok = rabbit_ct_broker_helpers:start_node(Config, 1), + + timer:sleep(3000), + + ?assertEqual(true, rabbit_ct_broker_helpers:rpc(Config, 1, + rabbit_vhost_sup_sup, is_vhost_alive, [VHost2])). + +node_starts_with_dead_vhosts_with_mirrors(Config) -> + VHost1 = <<"vhost1">>, + VHost2 = <<"vhost2">>, + + set_up_vhost(Config, VHost1), + set_up_vhost(Config, VHost2), + + true = rabbit_ct_broker_helpers:rpc(Config, 1, + rabbit_vhost_sup_sup, is_vhost_alive, [VHost1]), + true = rabbit_ct_broker_helpers:rpc(Config, 1, + rabbit_vhost_sup_sup, is_vhost_alive, [VHost2]), + [] = rabbit_ct_broker_helpers:rpc(Config, 1, + rabbit_vhost_sup_sup, check, []), + + Conn = rabbit_ct_client_helpers:open_unmanaged_connection(Config, 0, VHost1), + {ok, Chan} = amqp_connection:open_channel(Conn), + + QName = <<"node_starts_with_dead_vhosts_with_mirrors-q-0">>, + amqp_channel:call(Chan, #'queue.declare'{queue = QName, durable = true}), + ok = rabbit_ct_broker_helpers:rpc(Config, 0, + rabbit_policy, set, + [VHost1, <<"mirror">>, <<".*">>, [{<<"ha-mode">>, <<"all">>}], + 0, <<"queues">>, <<"acting-user">>]), + + %% Wait for the queue to start a mirror + timer:sleep(500), + + rabbit_ct_client_helpers:publish(Chan, QName, 10), + + {ok, Q} = rabbit_ct_broker_helpers:rpc( + Config, 0, + rabbit_amqqueue, lookup, + [rabbit_misc:r(VHost1, queue, QName)], infinity), + + Node1 = rabbit_ct_broker_helpers:get_node_config(Config, 1, nodename), + + [Pid] = amqqueue:get_sync_slave_pids(Q), + + Node1 = node(Pid), + + DataStore1 = rabbit_ct_broker_helpers:rpc( + Config, 1, rabbit_vhost, msg_store_dir_path, [VHost1]), + + rabbit_ct_broker_helpers:stop_node(Config, 1), + + file:write_file(filename:join(DataStore1, "recovery.dets"), <<"garbage">>), + + %% The node should start without a vhost + ok = rabbit_ct_broker_helpers:start_node(Config, 1), + + timer:sleep(3000), + + ?assertEqual(true, rabbit_ct_broker_helpers:rpc(Config, 1, + rabbit_vhost_sup_sup, is_vhost_alive, [VHost2])). + +vhost_creation_idempotency(Config) -> + VHost = <<"idempotency-test">>, + try + ?assertEqual(ok, rabbit_ct_broker_helpers:add_vhost(Config, VHost)), + ?assertEqual(ok, rabbit_ct_broker_helpers:add_vhost(Config, VHost)), + ?assertEqual(ok, rabbit_ct_broker_helpers:add_vhost(Config, VHost)) + after + rabbit_ct_broker_helpers:delete_vhost(Config, VHost) + end. + +%% ------------------------------------------------------------------- +%% Helpers +%% ------------------------------------------------------------------- + +open_connections(Config, NodesAndVHosts) -> + % Randomly select connection type + OpenConnectionFun = case ?config(connection_type, Config) of + network -> open_unmanaged_connection; + direct -> open_unmanaged_connection_direct + end, + Conns = lists:map(fun + ({Node, VHost}) -> + rabbit_ct_client_helpers:OpenConnectionFun(Config, Node, + VHost); + (Node) -> + rabbit_ct_client_helpers:OpenConnectionFun(Config, Node) + end, NodesAndVHosts), + timer:sleep(500), + Conns. + +close_connections(Conns) -> + lists:foreach(fun + (Conn) -> + rabbit_ct_client_helpers:close_connection(Conn) + end, Conns), + timer:sleep(500). + +count_connections_in(Config, VHost) -> + count_connections_in(Config, VHost, 0). +count_connections_in(Config, VHost, NodeIndex) -> + timer:sleep(200), + rabbit_ct_broker_helpers:rpc(Config, NodeIndex, + rabbit_connection_tracking, + count_tracked_items_in, [{vhost, VHost}]). + +set_up_vhost(Config, VHost) -> + rabbit_ct_broker_helpers:add_vhost(Config, VHost), + rabbit_ct_broker_helpers:set_full_permissions(Config, <<"guest">>, VHost), + set_vhost_connection_limit(Config, VHost, -1). + +set_vhost_connection_limit(Config, VHost, Count) -> + set_vhost_connection_limit(Config, 0, VHost, Count). + +set_vhost_connection_limit(Config, NodeIndex, VHost, Count) -> + Node = rabbit_ct_broker_helpers:get_node_config( + Config, NodeIndex, nodename), + ok = rabbit_ct_broker_helpers:control_action( + set_vhost_limits, Node, + ["{\"max-connections\": " ++ integer_to_list(Count) ++ "}"], + [{"-p", binary_to_list(VHost)}]). + +expect_that_client_connection_is_rejected(Config) -> + expect_that_client_connection_is_rejected(Config, 0). + +expect_that_client_connection_is_rejected(Config, NodeIndex) -> + {error, _} = + rabbit_ct_client_helpers:open_unmanaged_connection(Config, NodeIndex). + +expect_that_client_connection_is_rejected(Config, NodeIndex, VHost) -> + {error, _} = + rabbit_ct_client_helpers:open_unmanaged_connection(Config, NodeIndex, VHost). |